From f60ba488296b717407eca4a78df59b71dc43b7e3 Mon Sep 17 00:00:00 2001 From: Harish Chandrashekar Date: Tue, 14 Jul 2026 12:12:51 +0000 Subject: [PATCH] Data Validator for Custom UN Imports --- scripts/un/un_dataset_validator/README.md | 90 ++++++ .../un/un_dataset_validator/requirements.txt | 1 + .../rules/consolidated_rules.md | 283 ++++++++++++++++++ .../rules/rule_10_1_coded_attributes.md | 23 ++ .../rules/rule_10_attributes_as_columns.md | 83 +++++ .../rules/rule_11_statvar_dcid.md | 41 +++ .../rules/rule_12_names.md | 59 ++++ .../rules/rule_13_1_statvar_name_template.md | 27 ++ .../rule_15_duplicate_statvar_properties.md | 15 + .../rules/rule_16_observation_completeness.md | 17 ++ .../rules/rule_17_name_constraints.md | 46 +++ .../rules/rule_18_mcf_single_dcid_node.md | 42 +++ .../rules/rule_1_mapping_validation.md | 24 ++ .../rules/rule_2_series_to_populationtype.md | 68 +++++ .../rules/rule_3_geography.md | 101 +++++++ .../rules/rule_4_time_period.md | 82 +++++ .../rules/rule_5_obs_value_to_value.md | 70 +++++ .../rule_6_1_dimension_mapping_prefix.md | 20 ++ .../rules/rule_6_2_dimension_value_mapping.md | 96 ++++++ .../rules/rule_6_dimensions_and_attributes.md | 96 ++++++ .../rules/rule_7_unit_multiplier.md | 84 ++++++ .../rules/rule_8_unit_measure_mapping.md | 19 ++ .../rule_9_frequency_to_observationperiod.md | 67 +++++ .../run_custom_dataset.sh | 41 +++ .../scripts/base_validator.py | 37 +++ .../scripts/run_isolated_sdg_test.py | 88 ++++++ .../scripts/run_validations.py | 89 ++++++ .../scripts/summary_generator.py | 149 +++++++++ .../scripts/test_rule_1.py | 128 ++++++++ .../scripts/test_rule_10.py | 168 +++++++++++ .../scripts/test_rule_11.py | 94 ++++++ .../scripts/test_rule_12.py | 149 +++++++++ .../scripts/test_rule_13.py | 116 +++++++ .../scripts/test_rule_14.py | 111 +++++++ .../scripts/test_rule_15.py | 116 +++++++ .../scripts/test_rule_16.py | 147 +++++++++ .../scripts/test_rule_17.py | 136 +++++++++ .../scripts/test_rule_2.py | 148 +++++++++ .../scripts/test_rule_3.py | 206 +++++++++++++ .../scripts/test_rule_4.py | 154 ++++++++++ .../scripts/test_rule_5_7.py | 191 ++++++++++++ .../scripts/test_rule_6.py | 236 +++++++++++++++ .../scripts/test_rule_8.py | 226 ++++++++++++++ .../scripts/test_rule_9.py | 193 ++++++++++++ .../scripts/validator_utils.py | 51 ++++ .../SDG_q1-2026_OBS_AG_FLS_PCT_data.csv | 150 ++++++++++ ..._q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf | 35 +++ .../validation_implementation_analysis.md | 98 ++++++ 48 files changed, 4711 insertions(+) create mode 100644 scripts/un/un_dataset_validator/README.md create mode 100644 scripts/un/un_dataset_validator/requirements.txt create mode 100644 scripts/un/un_dataset_validator/rules/consolidated_rules.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_10_1_coded_attributes.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_10_attributes_as_columns.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_11_statvar_dcid.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_12_names.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_13_1_statvar_name_template.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_15_duplicate_statvar_properties.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_16_observation_completeness.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_17_name_constraints.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_18_mcf_single_dcid_node.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_1_mapping_validation.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_2_series_to_populationtype.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_3_geography.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_4_time_period.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_5_obs_value_to_value.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_6_1_dimension_mapping_prefix.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_6_2_dimension_value_mapping.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_6_dimensions_and_attributes.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_7_unit_multiplier.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_8_unit_measure_mapping.md create mode 100644 scripts/un/un_dataset_validator/rules/rule_9_frequency_to_observationperiod.md create mode 100755 scripts/un/un_dataset_validator/run_custom_dataset.sh create mode 100644 scripts/un/un_dataset_validator/scripts/base_validator.py create mode 100644 scripts/un/un_dataset_validator/scripts/run_isolated_sdg_test.py create mode 100644 scripts/un/un_dataset_validator/scripts/run_validations.py create mode 100644 scripts/un/un_dataset_validator/scripts/summary_generator.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_1.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_10.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_11.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_12.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_13.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_14.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_15.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_16.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_17.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_2.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_3.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_4.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_5_7.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_6.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_8.py create mode 100644 scripts/un/un_dataset_validator/scripts/test_rule_9.py create mode 100644 scripts/un/un_dataset_validator/scripts/validator_utils.py create mode 100644 scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data.csv create mode 100644 scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf create mode 100644 scripts/un/un_dataset_validator/validation_implementation_analysis.md diff --git a/scripts/un/un_dataset_validator/README.md b/scripts/un/un_dataset_validator/README.md new file mode 100644 index 0000000000..ec75f8e03a --- /dev/null +++ b/scripts/un/un_dataset_validator/README.md @@ -0,0 +1,90 @@ +# UN Data Commons - Dataset Validation Suite + +This directory contains the Python-based validation suite used to ensure datasets conform to the UN Data Commons Schema mapping rules before ingestion. + +## Prerequisites + +1. **Python 3:** Ensure Python 3.8+ is installed on your system. +2. **Required Libraries:** Install the dependencies required by the validation scripts: + ```bash + cd un_dataset_validator + pip install -r requirements.txt + ``` + +## Directory Structure Expectations + +To run the validation successfully, your datasets and configurations must be organized into specific nested folders. The validation suite requires paths to **three** distinct directories: + +### 1. Processed Directory (``) +This is the main directory containing your custom dataset's final generated mappings and outputs. It **must** follow this nested folder structure: + +```text +my_processed_dir/ <-- This is the you provide +├── processed_data/ <-- (Required) Contains your final processed *_data.csv files +│ ├── SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv +│ └── SDG_q1-2026_OBS_AG_FLS_PCT_data.csv +├── schema/ <-- (Required) Contains your generated schemas +│ ├── SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf +│ └── SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf +├── pvmap/ <-- (Optional) Contains local Property-Value maps for this dataset +│ └── CL_UNIT_MEASURE_pvmap.csv +└── dc_generated/ <-- (Optional) Intermediate generated outputs +``` +*Note: If your dataset relies on global mapping files (like a global `un_geography_pvmap.csv`), the suite will automatically look for a `pvmap/` folder in the **parent** directory of your `` (e.g., `../my_processed_dir/../pvmap/`).* + +### 2. Raw Input Data Directory (``) +This folder contains the original, raw CSV files (the data you processed into the `processed_data` folder). It is critical that the base names correspond to the processed files. +```text +raw_data/ +└── DATA/ <-- This is the you provide + ├── SDG_q1-2026_OBS_AG_FLS_INDEX.csv + └── SDG_q1-2026_OBS_AG_FLS_PCT.csv +``` + +### 3. DSD Schema Directory (``) +This folder contains the SDMX Data Structure Definition (DSD) files used to validate your dimensions and attributes. The files typically have `_DSD_` in their names. +```text +raw_data/ +└── DSD/ <-- This is the you provide + ├── SDG_q1-2026_DSD_AG_FLS_INDEX.csv + └── SDG_q1-2026_DSD_AG_FLS_PCT.csv +``` + +## How to Run Validations + +The easiest way to run the full validation suite on your custom dataset is using the provided shell wrapper script: + +```bash +cd un_dataset_validator +./run_custom_dataset.sh +``` + +### Example Usage: + +Assume you have a project directory organized like this: +```text +my_project/ +├── raw_inputs/ +│ ├── DATA/ <-- Raw CSVs +│ └── DSD/ <-- Raw SDMX Schemas +├── processed/ +│ └── health_v1/ <-- Processed Dataset Folder +│ ├── processed_data/ +│ └── schema/ +└── un_dataset_validator/ <-- This validation suite +``` + +To run the validation on `health_v1`, you would execute: + +```bash +cd un_dataset_validator +./run_custom_dataset.sh health_v1 ../processed/health_v1 ../raw_inputs/DATA ../raw_inputs/DSD +``` + +### Output & Logs + +The script will automatically create a dedicated log directory inside `un_dataset_validator/logs/` named `_validation_logs/`. + +Inside this folder, you will find: +1. Individual `ruleX_..._validation.log` files containing detailed tracing of each rule's execution. +2. A final **`summary.md`** file providing a clean, comprehensive markdown matrix of which rules passed/failed alongside detailed metrics and error samples. diff --git a/scripts/un/un_dataset_validator/requirements.txt b/scripts/un/un_dataset_validator/requirements.txt new file mode 100644 index 0000000000..fb335a7eea --- /dev/null +++ b/scripts/un/un_dataset_validator/requirements.txt @@ -0,0 +1 @@ +pandas>=1.0.0 \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/consolidated_rules.md b/scripts/un/un_dataset_validator/rules/consolidated_rules.md new file mode 100644 index 0000000000..40bc2e600f --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/consolidated_rules.md @@ -0,0 +1,283 @@ +# UN Dataset Validation Rules Consolidated Guide + +This document consolidates all dataset validation rules used by the UN Data Commons validator. It serves as a single, cohesive, and readable reference for the schema mapping, format requirements, and validation checks enforced across datasets from various agencies (e.g., SDG, ILO, UNICEF). + +--- + +## Rule 1: 1:1 Mapping Validation (PV Maps) + +### Requirement +Ensure all UN concepts and codes have a strict 1:1 mapping with the agency-specific Data Commons Project (DCP) schema. + +### Core Explanation +* **No Duplicate Properties:** There must be no duplicate properties assigned to a single concept. +* **Property-Value (PV) Alignment:** Within the PV maps, the `event code` column must align perfectly with the `constraint property` column. For example, if the concept is "age" or "poverty status", the `event code` must be "age" or "poverty status" exactly, maintaining consistency across the file. +* **Scope:** This rule applies strictly to the agency-specific schema rather than the base Data Commons mapping. + +### Implementation & Verification Logic +1. **Target Files:** Locate and parse PV map files (e.g., `CL_*.csv` or other PV maps in the agency's schema folder). +2. **Canonical Matching:** The validation script uses a canonical format helper (stripping spaces, underscores, and standardizing case) to match `event code` (UnConcept) to `constraint property` (ConstraintProp) instead of requiring exact string equality. +3. **Core Exemptions:** The core concepts `series`, `geography`, `timeperiod`, and `obsvalue` are explicitly skipped during this 1:1 mapping check. + +--- + +## Rule 2: SERIES Mapping to `populationType` + +### Requirement +The `series` identifier from the input file must be mapped to the `populationType` property on the generated Statistical Variable nodes in the output `.mcf` file, prefixed with the responsible agency's identifier. + +### Core Explanation +* **Series Source:** The series identifier is derived from the input filename (e.g., `AG_FLS_INDEX` from `SDG_..._AG_FLS_INDEX_data.csv`) or an explicit column within the input CSV. +* **Agency-Specific Prefix:** The series code in the `populationType` must be formatted as: `UN__SERIES-`. + * *Example:* `populationType: dcid:UN_SDG_SERIES-AG_FLS_INDEX` +* **Case Sensitivity:** Convert the agency name to uppercase (e.g., `sdg` -> `SDG`) before forming the expected DCID. + +### Implementation & Verification Logic +1. **Determine Context:** Identify the agency name (e.g., `SDG`, `ILO`, `UNICEF`) based on the root directory of the dataset. +2. **Validate MCF Nodes:** Iterate through all parsed nodes in the output MCF where `typeOf: dcid:StatisticalVariable`. +3. **Assert Property:** Ensure `populationType` exists and is equal to `dcid:UN__SERIES-`. Skip non-StatVar nodes to prevent false positives. + +--- + +## Rule 3: GEOGRAPHY Mapping to `observationAbout` + +### Requirement +The geographical identifiers found in the input dataset's geography columns must be mapped to the `observationAbout` column in the generated output `.csv` file. Geographical entities at the "Country" level or above must be successfully resolved to a Data Commons identifier (DCID). + +### Core Explanation +* **Geography PV Map:** UN geographical codes are translated into valid DCIDs via the central mapping file `/all_data/pvmap/un_geography_pvmap.csv`. +* **Unresolved National Geographies (Rule 3.1):** If a geography code representing a country-level or higher place (e.g., countries, continents, global regions, Earth) cannot be resolved (or is marked as `#ignore`/`unmapped place`), it must be logged in an explicit error report (`missing_national_geographies.log`). + +### Implementation & Verification Logic +1. **Load Reference Files:** Load `un_geography_pvmap.csv` for DCID lookup and `/all_data/un_geography.csv` to understand geographical hierarchies and names. +2. **Column Matching:** Validate that geography codes in the input column (e.g., `geo`, `REF_AREA`) match the expected DCID in the output CSV's `observationAbout` column. +3. **Row Alignment:** Use the output CSV's `#input` lineage column rather than raw indices to safely join and align rows between input and output files. + +--- + +## Rule 4: Time Period to observationDate + +### Requirement +Ensure that the `TIME_PERIOD` (or `timePeriod`) column in the source data accurately maps to the Data Commons `observationDate` property in the output CSV, utilizing conversion rules for complex or non-standard formats. + +### Core Explanation +The data pipeline standardizes non-standard date formats into correct observation dates based on three validation levels: +1. **YYYY Format:** If the source matches a standard year, the validation checks if the `observationDate` matches the first 4 characters (YYYY) of the input. +2. **Interval Start Date Extraction:** If the input time format is an ISO-8601 interval (e.g., `2014-01/P3M`), the pipeline extracts the start date component (`2014-01`) as the `observationDate`. +3. **Fallback Passthrough:** If conversion checks fail, the value is passed through unaltered, and validation asserts exact string equivalence. + +--- + +## Rule 5: OBS_VALUE Mapping to `value` + +### Requirement +The observation value (`OBS_VALUE` column) from the input data must be mapped to the `value` property in the output CSV as a valid number (`value.number`). + +### Core Explanation +* **Format Requirements:** The final value in the output must be formatted as a clean numeric value (e.g., `"1,000.50"` becomes `1000.50`) and must be parseable as a float. +* **Empty/Missing Values:** Missing or empty observation entries are mapped to empty/missing in the output rather than failing the numeric check. +* **Multiplier Interaction:** The final value in the output may be adjusted if unit multipliers apply (see Rule 7), but the requirement that the resulting value is a valid `{Number}` remains absolute. + +--- + +## Rule 6: Dimensions vs. Attributes + +### Requirement +All properties defined in the Dataset Definition (DSD) file must be correctly handled as either a **dimension** or an **attribute**. + +### Core Explanation +* **Dimensions:** Except for specific exemptions (`Geography`, `Time Period`, `OBS_VALUE`, and `SERIES`), any property marked as a "dimension" must be attached to the Statistical Variable (StatVar) in the `.mcf` file as a **constraint property** (e.g., `product: dcid:UN_PRODUCT-X`). +* **Attributes:** Any property marked as an "attribute" (e.g., `CENSORED_VALUE_TYPE`, `FOOTNOTE`) must **NOT** be attached to the Statistical Variable. Instead, they must be included in the output data `.csv` file as separate columns (e.g., `censoredValueType`). + +### Implementation & Verification Logic +1. **Parse DSD:** Load the DSD file and separate concepts by their `ROLE` column (dimension vs. attribute), ignoring the standard exemptions. +2. **Verify CSV Columns:** Ensure all attribute concepts appear as separate columns in the output CSV, and do not exist as properties inside MCF nodes. +3. **Verify MCF Constraints:** Ensure all non-exempt dimensions are attached to the corresponding Statistical Variable nodes as constraints. + +--- + +## Rule 6.1: Dimension Mapping to DCP Schema with UN_ Prefix + +### Requirement +Each dimension must be mapped to the Data Commons Project (DCP) schema using a common prefix `UN_` without including agency-specific prefixes. + +### Core Explanation +* **Consistency:** Any schema generated from the DCP must consistently use the `UN_` prefix (e.g., `UN_PRODUCT-...`) instead of introducing agency-specific identifiers in the prefix (e.g., `UN_ECLAC_` or `UN_ILO_`). +* **Status:** This validation check is **temporarily ignored/skipped** in the validator scripts pending further clarification. + +--- + +## Rule 6.2: Dimension Value Mapping + +### Requirement +Every value within a dimension column must be properly mapped to a generated property value DCID following the exact template: `_-`. + +### Core Explanation +* **Template Structure:** The expected format is `_-`, where: + * `` is typically `UN` (or configured per dataset). + * `` is the capitalized dimension column name (e.g., `PRODUCT`). + * `` is the raw value from the input. + * *Example:* `UN_PRODUCT-CPC2_1_0113` +* **Special Character Handling:** Any special characters or spaces in the raw value must be converted to underscores (`_`) before constructing the DCID. +* **Null Safety:** Missing or null values are ignored to prevent invalid strings (like `UN_PRODUCT-nan`). + +--- + +## Rule 7: UNIT_MULTIPLIER Application + +### Requirement +The observation value must be scaled and verified in the output dataset using the specified unit multiplier factor. + +### Core Explanation +* **Lookup Factor:** The validator uses the common PV map `all_data/pvmap/CL_MULT_pvmap_multiply.csv` to resolve the numeric multiplication factor associated with the multiplier code (e.g., code `-15` maps to `1.00E-15`). +* **Multiplication Logic:** The expected output value is calculated as: + `Expected_Value = float(Raw_OBS_VALUE) * float(Multiplier_Factor)` +* **Validation Check:** Assert that the calculated `Expected_Value` matches the `value` column in the output CSV, allowing for minor floating-point drift. + +--- + +## Rule 8: UNIT_MEASURE Mapping and Multiplier Logic Integration + +### Requirement +`UNIT_MEASURE` must be mapped to an existing Data Commons Project (DCP) enum using the name from the source dataset, and its validation must be closely integrated with the multiplier logic. + +### Core Explanation +* **Enum Mapping:** Verify that each `UNIT_MEASURE` from the source maps correctly to a valid DCP enum corresponding to the unit defined in the source data. +* **Multiplier Integration:** Connect the validation of unit measures with unit multipliers to check that when a unit is applied, any corresponding multiplier scaling (e.g., scaling the value based on the multiplier enum) is also evaluated and applied correctly. + +--- + +## Rule 9: Frequency to observationPeriod + +### Requirement +Ensure that the `FREQUENCY` column in the source data correctly maps to the Data Commons `observationPeriod` property in the output CSV, according to the common Property-Value (PV) map. + +### Core Explanation +* **Lookup Logic:** Cross-reference the input frequency code (e.g., `A` for Annual, `M` for Monthly, `Q` for Quarterly) with the `UnCode` column in `CL_FREQUENCY_pvmap_obsperiod.csv` to find the mapped observation period (e.g., `P1Y` for Annual). +* **Verification:** Verify that the resolved observation period is present in the `observationPeriod` column of the corresponding output CSV row. +* **Header Typo Handling:** The pipeline has been known to generate files with the header `opservationPeriod` instead of `observationPeriod`. The validator dynamically checks for this typo, logs it, and flags it as a failure if the correct header is missing. + +--- + +## Rule 10: Attributes as Output Columns + +### Requirement +Every concept defined in the dataset's Data Structure Definition (DSD) file with a `ROLE` of `Attribute` must be present as a separate column in the output `data.csv` file. + +### Core Explanation +* **Supplementary Data:** Attributes provide metadata (such as footnotes, observation statuses, or flags) and must NOT be attached to the Statistical Variables in the MCF. +* **Validation:** Filter the DSD for `ROLE='Attribute'`, extract their identifiers (e.g., `OBS_STATUS`, `FOOTNOTE`), and verify they exist as distinct columns in the output CSV header. +* **Exclusions (Rule 10.1):** Parsing and validating the internal string structures of complex attributes (e.g., verifying multiple comma-separated footnotes inside a single `FOOTNOTE` cell) is excluded from this validation check. + +--- + +## Rule 10.1: Coded Attributes Mapping to Property Enum + +### Requirement +All coded attributes within the dataset (attributes that reference a Code List / CL file) must be correctly mapped and assigned a `property:enum` value in the Data Commons Project (DCP) schema. + +### Core Explanation +* **Schema Integrity:** Coded attributes pulling from a restricted set of values must reflect this in the schema by assigning a `property:enum` mapping to maintain structural integrity. +* **Status:** This validation check is **temporarily ignored/skipped** in the validator scripts pending further discussion. + +--- + +## Rule 11: StatVar DCID Format & Special Characters + +### Requirement +Validates that the Data Commons Identifier (DCID) generated for each Statistical Variable (StatVar) in the output `.mcf` matches the expected hierarchical template, and any illegal or special characters within the originating codes are converted to underscores (`_`). + +### Core Explanation +* **DCID Template:** `//[.--__…]` + * *Example:* `dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD` +* **Special Character Conversion:** Any spaces, dashes, slashes, or parentheses in the original series, concept, or code must be converted to underscores (`_`) before constructing the DCID string. +* **Regex Verification:** The validator applies a regular expression to enforce template structural integrity. It is flexible on the prefix and agency and expects underscores (`_`) as word separators rather than hyphens. + +--- + +## Rule 12: Names (alternateName, Value names, nameWithLanguage) + +### Requirement +Ensure that names for properties and values generated in the schema accurately reflect their source definitions in the DSD and Codelist files. + +### Core Explanation +* **Validation Checks:** + * **Rule 12.1 (Property Names):** A property's `alternateName` must match the corresponding name defined in the DSD file. + * **Rule 12.2 (Value Names):** A value's name must match the name defined in the specific concept's codelist (`CL` file). + * **Rule 12.3 (Multi-lingual Names):** Translations available in other languages must be added to the `nameWithLanguage` property. +* **String Normalization:** To prevent false mismatches from spacing or punctuation quirks, a `normalize_name()` function strips all punctuation, converts strings to lowercase, and normalizes spacing before comparison. +* **Omissions:** Due to a pipeline issue where `name` properties are omitted from `StatisticalVariable` nodes, the validator currently logs a warning rather than a failure for missing StatVar names. + +--- + +## Rule 13.1: Statvar Name Template + +### Requirement +The name assigned to a Statistical Variable (StatVar) must adhere to a specific template format to ensure consistency and readability. + +### Core Explanation +* **Template Structure:** `" [=, ...]"` + * *Example:* `"Unemployment Rate [Age=15-24, Gender=Female]"` +* **Formatting Rules:** Uses square brackets to enclose constraints, comma-space to separate multiple pairs, and equals sign to separate concept and code names. +* **Status:** This validation is **currently skipped (passed with warnings)** because the pipeline does not reliably generate the `name` property for `StatisticalVariable` nodes. + +--- + +## Rule 15: Duplicate Properties in Statistical Variables + +### Requirement +No property key or concept code is allowed to be repeated twice within the same Statistical Variable (StatVar) definition. + +### Core Explanation +* **Structural Consistency:** Duplicating dimension values violates the Data Commons structural requirements and can lead to inconsistent behavior downstream. +* **Validation Logic:** The script scans `.mcf` (or `.tmcf`) files for `Node: dcid:...` blocks, tracks all property keys defined inside each block, and flags an error if any property key is defined more than once in the same node. + +--- + +## Rule 16: Observation Completeness (#input tracking) + +### Requirement +Guarantees that no non-empty observations are improperly dropped during processing by tracing every valid input cell to an `#input` cell lineage reference in the output files. + +### Core Explanation +* **Lineage Tracking:** The output CSV files contain an `#input` column in the format `filename:row:col` tracking the origin of each data point. +* **Validation Logic:** + 1. Parse and collect all `#input` cell references from generated output CSV files. + 2. Scan every raw input CSV file to identify cells containing non-empty observation values (e.g., in `OBS_VALUE`). + 3. Ensure that every identified valid input cell coordinates exist within the collected `#input` references. + +--- + +## Rule 17: Name Constraints (Bracket and Code Prohibitions) + +### Requirement +Ensures that the generated `name` properties in the dataset's schemas are human-readable, descriptive, properly formatted, and free of technical prefixes, file identifiers, or template artifacts. + +### Core Explanation +* **Bracket Check (Rule 17.1):** A node's name must not start with a bracket `[` (ignoring leading whitespace), ensuring names are not raw template indicators. +* **Code Concept Check (Rule 17.2):** A node's name must not contain technical identifier codes, file prefixes (such as `CL_`, `DSD_`, `FSP`, `TFT`), or uppercase technical values containing underscores (e.g., `ISCED11_02`, `AGG_ANIMAL_PROD`). +* **Exemptions & Safeguards:** + * Skipping name checks for `StatVarGroup` nodes since their names naturally contain taxonomic labels. + * Protecting descriptive acronyms (e.g., `GDP`, `FDI`, `UNCLOS`, `CO2`, `ISIC4`, `MGCI`) from triggering false positives. + +--- + +## Rule 18: Single-DCID Node Declarations and MCF Comma Prevention + +### Requirement +Every `Node:` statement in MCF files must declare exactly one valid Data Commons Identifier (DCID) and must **never** contain commas (`,`). When representing hierarchical links or relationships using properties like `specializationOf:` or `memberOf:`, they must reference the correct consolidated parent DCID (using double-underscore `__` joiners) instead of incorrectly separating them with commas. + +### Core Explanation +* **Origin & Context:** This issue was first identified in `SDG_statvars_groups.mcf` but can occur in similar MCF schema or StatVar group files across any other dataset. It usually stems from automated pipeline scripts incorrectly joining multiple concepts. +* **No Commas in Node Declarations:** Commas are invalid in a `Node:` line because each node definition block represents exactly one entity. Commas (such as `,dcid:`) in a `Node:` statement will crash MCF parsers. +* **Consolidated DCID Joiner (`__`):** Multidimensional or combined keys must use double underscores (`__`) to join constituent DCIDs within a single identifier rather than commas. + * *Example (Invalid Node):* `Node: dcid:A,dcid:B,dcid:C` + * *Example (Valid Node):* `Node: dcid:A__B__C` +* **Hierarchy Link Alignment (`specializationOf:` / `memberOf:`):** While commas are technically syntactically valid in `specializationOf:` or `memberOf:` properties to specify multiple distinct parent references, they must not be used to reference parts of a consolidated parent. + * If a parent is defined as `dcid:A__B__C`, the child node's relationship must refer to `specializationOf: dcid:A__B__C`. + * If the relationship is defined with commas as `specializationOf: dcid:A,dcid:B,dcid:C`, the parser will interpret it as three separate references to individual nodes (`dcid:A`, `dcid:B`, and `dcid:C`), none of which match the consolidated parent `dcid:A__B__C`, breaking the hierarchy link. + +### Implementation & Verification Logic +1. **Assert No Commas in Node Values:** The validator scans every MCF file, identifies all lines beginning with `Node:`, and ensures no comma `,` is present in the DCID value string. + * *Validation assertion:* `assert ',' not in node_dcid_str, f'Malformed Node name with "," at line {line_num}'` +2. **Synchronized Parent-Child Check:** Ensure that any `specializationOf:` or `memberOf:` lines referencing consolidated parents use double underscores `__` to match the corrected parent node's consolidated DCID. diff --git a/scripts/un/un_dataset_validator/rules/rule_10_1_coded_attributes.md b/scripts/un/un_dataset_validator/rules/rule_10_1_coded_attributes.md new file mode 100644 index 0000000000..2c63f10394 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_10_1_coded_attributes.md @@ -0,0 +1,23 @@ +# Rule 10.1: Coded Attributes Mapping to Property Enum + +## Requirement +All coded attributes within the dataset must be correctly mapped and assigned a `property:enum` value in the Data Commons Project (DCP) schema. + +## Context & Rules +- According to Rule 10, all attributes (columns marked as 'Attribute' in the DSD) should become output columns along with the observation. +- For attributes that are specifically *coded* (meaning they pull from a defined codelist or restricted set of values, as opposed to free-text), the schema must reflect this by assigning a `property:enum` mapping. +- This ensures that coded attributes maintain their structural integrity and defined value set in the output DCP schema. + +## Implementation Logic +1. **Target Files**: Data Structure Definition (DSD) files, source data files, and schema mapping files. +2. **Identification of Coded Attributes**: + - Parse the DSD file to identify columns where the ROLE is defined as 'Attribute'. + - Determine which of these attributes are "coded" (i.e., they reference a Code List / CL file). +3. **Enum Mapping Validation**: + - Verify that for every coded attribute identified, the resulting schema generation logic assigns it a `property:enum` value mapping. + - Check the output files to ensure that the schema correctly reflects the enum type for these specific attributes, while leaving text attributes as raw values. +4. **Error Flagging**: + - If a coded attribute is found that is NOT mapped to a `property:enum` in the schema (e.g., if it is mapped as a plain text string), flag this as a validation error. + +## Implementation Deviations +- **Temporarily Excluded:** During the June 23rd meeting, it was explicitly decided that all validations enforcing base DCP schema mappings are to be temporarily ignored pending further discussion with AJ. The validation script currently skips this check to align with this decision. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_10_attributes_as_columns.md b/scripts/un/un_dataset_validator/rules/rule_10_attributes_as_columns.md new file mode 100644 index 0000000000..1887d56305 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_10_attributes_as_columns.md @@ -0,0 +1,83 @@ +# Rule 10: Attributes as Output Columns + +## Description +According to the UN Data Commons mapping rules, every concept defined in the dataset's Data Structure Definition (DSD) file with a `ROLE` of `Attribute` must be present as a separate column in the output `data.csv` file. + +Attributes provide supplementary information (such as footnotes, observation statuses, or flags) about the observation value. Unlike dimensions, attributes must **NOT** be attached as properties to the Statistical Variables. + +**Note on Exclusions (Rule 10.1):** Parsing and validating the internal string structures of complex attributes (e.g., verifying comma-separated multiple footnotes inside a single `FOOTNOTE` cell) is marked as an "Ask Ajai" item and is strictly excluded from this validation check. This rule only validates the *presence* of the attribute column in the output. + +## Files Involved +- **Input:** Dataset-specific DSD file (e.g., `schema/dsd.csv` or similar file defining `ROLE`). +- **Output:** The generated data CSV file (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv`). + +## Validation Logic +1. **Identify Attributes:** Parse the DSD file and filter for all rows where the `ROLE` column is equal to `Attribute` (case-insensitive). +2. **Extract Identifiers:** Extract the concept identifier/name for each attribute (e.g., `OBS_STATUS`, `FOOTNOTE`). +3. **Verify Output Columns:** Read the header of the generated output `data.csv` file. +4. **Compare:** Ensure that every attribute identified in step 1 exists as a distinct column in the output CSV header. +5. **Report Missing Columns:** If an attribute defined in the DSD is missing from the output CSV, flag it as a validation failure. + +## Python Implementation + +```python +import pandas as pd +import glob +import os + +def validate_rule_10(dsd_file_path: str, output_csv_path: str) -> bool: + """ + Validates that all concepts with ROLE='Attribute' in the DSD are present as + columns in the output CSV. + """ + print(f"Validating Rule 10: Attributes as Output Columns for {os.path.basename(output_csv_path)}") + + try: + # 1. Read DSD and find Attributes + dsd_df = pd.read_csv(dsd_file_path) + + # Ensure required columns exist + # Note: Actual column names for Concept/Identifier and Role might vary slightly + # Adjust 'concept' and 'role' based on the exact DSD schema structure. + role_col = next((c for c in dsd_df.columns if c.strip().lower() == 'role'), None) + concept_col = next((c for c in dsd_df.columns if c.strip().lower() in ['concept', 'id', 'name']), None) + + if not role_col or not concept_col: + print(f" [ERROR] DSD file missing 'ROLE' or Concept identifier column.") + return False + + # Filter attributes and get their names + attributes = dsd_df[dsd_df[role_col].str.lower() == 'attribute'][concept_col].dropna().tolist() + # Clean up strings + expected_attribute_cols = [attr.strip() for attr in attributes] + + if not expected_attribute_cols: + print(" [INFO] No attributes found in DSD. Validation passed.") + return True + + # 2. Read Output CSV headers + output_df = pd.read_csv(output_csv_path, nrows=0) # Read only header + output_columns = [col.strip() for col in output_df.columns] + + # 3. Verify presence + missing_attributes = [] + for attr in expected_attribute_cols: + # Check exact match or case-insensitive match based on pipeline behavior + match_found = any(attr.lower() == col.lower() for col in output_columns) + if not match_found: + missing_attributes.append(attr) + + if missing_attributes: + print(f" [FAILURE] Missing Attribute columns in output CSV: {missing_attributes}") + return False + + print(" [SUCCESS] All DSD Attributes are present as columns in the output CSV.") + return True + + except Exception as e: + print(f" [ERROR] Validation failed due to exception: {e}") + return False + +# Example Usage: +# validate_rule_10('schema/dsd.csv', 'extracted_data_new/20260615/SDG_q1-2026_OBS_AG_FOOD_WST_data.csv') +``` diff --git a/scripts/un/un_dataset_validator/rules/rule_11_statvar_dcid.md b/scripts/un/un_dataset_validator/rules/rule_11_statvar_dcid.md new file mode 100644 index 0000000000..98f3b84893 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_11_statvar_dcid.md @@ -0,0 +1,41 @@ +# Rule 11: StatVar DCID Format & Special Characters + +## Overview +This rule validates the structural format of the Data Commons Identifier (DCID) generated for each Statistical Variable (StatVar). The DCID must strictly adhere to a specific templated format, and any illegal or special characters within the originating codes must be converted to underscores (`_`) to ensure valid identifier syntax. + +## Files Involved +- **Output:** `output_stat_vars.mcf` (Specifically examining the `Node: dcid:...` lines). + +## Implementation Details + +### 1. DCID Template +The DCID must be constructed using the following template: +`//[.--__…]` + +Where: +- `` is generally `undata`. +- `` is the agency identifier (e.g., `sdg`). +- `` is the identifier of the series (e.g., `AG_FLS_INDEX`). +- The `[.--__…]` portion represents the dimensional constraints attached to the StatVar, chained together. + +### 2. Special Character Conversion +Any special characters present in the original ``, ``, or `` values (such as spaces, dashes, slashes, or parentheses) MUST be converted to underscores (`_`) before being concatenated into the DCID string. + +## Example +Given an agency of `sdg`, a series of `AG_FLS_INDEX`, a concept of `PRODUCT`, and a code of `AGG_ANIMAL_PROD`: + +**Expected MCF Node Definition:** +``` +Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD +``` + +## Python Implementation Strategy + +1. **Parse MCF:** Read the `output_stat_vars.mcf` file and extract all values from the `Node:` property that begin with `dcid:`. +2. **Regex Validation:** Use a regular expression to validate the structural integrity of the extracted DCID against the expected template. A simplified regex structure would look like: `^dcid:undata/[a-z0-9_-]+/[A-Z0-9_-]+(\.[A-Z0-9_-]+--[A-Z0-9_-]+(__[A-Z0-9_-]+--[A-Z0-9_-]+)*)?$`. (Note: The exact regex will need to be refined based on the full scope of allowed characters in standard DCIDs, but it must enforce the template hierarchy). +3. **Character Check:** Independently verify that no spaces or unescaped/unconverted special characters exist within the identifier string after the `dcid:` prefix. +4. **Reconciliation (Advanced):** If the input data is available during this check, independently reconstruct the expected DCID using the input series and dimension codes (applying the underscore conversion logic) and verify it matches the DCID found in the MCF. + +## Implementation Deviations +- **Flexible Agency/Prefix:** The implemented regular expression (`^dcid:[a-zA-Z0-9_]+/[a-zA-Z0-9_]+/...`) dynamically accepts any alphanumeric sequence for the prefix and agency, rather than hardcoding `undata`. +- **Hyphens vs. Underscores:** The strict regex implementation exclusively expects underscores (`_`) as word separators in series, concepts, and codes, differing from earlier examples that suggested hyphens (`-`) might be allowed. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_12_names.md b/scripts/un/un_dataset_validator/rules/rule_12_names.md new file mode 100644 index 0000000000..ae92a62bc0 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_12_names.md @@ -0,0 +1,59 @@ +# Rule 12: Names (alternateName, Value names, nameWithLanguage) + +## 1. Rule Description +* **Core Rule (12):** This rule ensures that names for properties and values generated in the schema accurately reflect their source definitions in the DSD (Data Structure Definition) and Codelists. +* **Sub-rule (12.1):** A property's `alternateName` must match the corresponding name defined in the DSD file. +* **Sub-rule (12.2):** A value's name must match the name defined in the specific concept's codelist (`CL` file). +* **Sub-rule (12.3):** Any names available in languages other than the default must be appropriately added to the `nameWithLanguage` property. + +## 2. Files Involved +* **Data Input:** The transcoded dataset. +* **DSD File:** Defines the structural metadata and concepts. +* **Codelists (CL):** Define the valid values and their corresponding names for each concept. +* **Output MCF Files:** The generated schema and stat vars where these properties are defined. + +## 3. Validation Logic & Flow +1. **Property Names (12.1):** + - Parse the generated schema. + - For each property, locate its corresponding concept in the DSD. + - Assert that the `alternateName` in the generated schema exactly matches the name string in the DSD. +2. **Value Names (12.2):** + - Parse the generated schema. + - For each value, identify its concept and find the corresponding codelist file. + - Assert that the value name in the generated schema matches the name in the codelist. +3. **Multi-lingual Names (12.3):** + - If the DSD or codelists provide names in multiple languages (e.g., using language tags), check that the generated schema utilizes the `nameWithLanguage` property correctly to represent these translations. + +## 4. Python Implementation Strategy (Draft) + +```python +import pandas as pd +# Pseudocode structure for Rule 12 Validation + +def validate_rule_12(schema_mcf_path: str, dsd_path: str, cl_dir_path: str) -> dict: + """ + Validates naming conventions against DSD and Codelists. + """ + errors = [] + + # 1. Load DSD for property names + # dsd_df = pd.read_csv(dsd_path) + + # 2. Parse MCF to extract properties and values + # ... + + # 3. Check 12.1: Property alternateName vs DSD + # ... + + # 4. Check 12.2: Value names vs Codelists + # ... + + # 5. Check 12.3: nameWithLanguage + # ... + + return {"status": "PASSED" if not errors else "FAILED", "errors": errors} +``` + +## Implementation Deviations +- **String Normalization:** To account for pipeline inconsistencies (such as varying quotes, spacing, or punctuation), the script implements a `normalize_name()` function. This strips all punctuation, converts strings to lowercase, and normalizes spacing before performing the string comparison. +- **Missing StatVar Names:** Due to a known pipeline issue where `name` properties are omitted from `StatisticalVariable` nodes, the validator currently logs a warning (rather than a failure) when this occurs. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_13_1_statvar_name_template.md b/scripts/un/un_dataset_validator/rules/rule_13_1_statvar_name_template.md new file mode 100644 index 0000000000..026bb1f8c8 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_13_1_statvar_name_template.md @@ -0,0 +1,27 @@ +# Rule 13.1: Statvar Name Template + +## Requirement +The name assigned to a Statistical Variable (StatVar) must adhere to a specific template format to ensure consistency and readability across the Data Commons Project (DCP). + +## Context & Rules +- According to the checklist, the required template for a StatVar name is: + `" [=, ...]"` +- This template provides a clear, human-readable summary of the underlying data series and the specific constraint properties (concepts and codes) that define the statistical variable. +- For example, if the series is "Unemployment Rate" and the concepts are "Age" and "Gender" with codes "15-24" and "Female" respectively, the name should be constructed as: + `"Unemployment Rate [Age=15-24, Gender=Female]"` + +## Implementation Logic +1. **Target Files**: Output MCF files where StatVars are defined (e.g., `output_stat_vars.mcf`), and relevant source data/DSD files to fetch names. +2. **Template Validation**: + - Extract the generated `name` property for each StatVar node. + - Deconstruct the underlying series name and its associated constraint concepts and codes. + - Verify that the generated name strictly matches the format: `" [=, ...]"` +3. **Format Checks**: + - Ensure the square brackets `[]` are used correctly to enclose the constraints. + - Ensure a comma and a space `, ` separate multiple concept=code pairs. + - Ensure an equals sign `=` separates the concept name and the code name. +4. **Error Flagging**: + - Flag a validation error if a StatVar name does not conform to this template or if the constituent parts (series name, concept name, code name) do not accurately reflect the underlying data definition. + +## Implementation Deviations +- **Skipped Execution:** This validation is currently skipped entirely by the script. Because the data pipeline does not reliably generate the `name` property for `StatisticalVariable` nodes (a known missing feature tracked under "Ask Ajai" rules), the script logs a warning and marks the validation as "PASSED (Skipped)". \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_15_duplicate_statvar_properties.md b/scripts/un/un_dataset_validator/rules/rule_15_duplicate_statvar_properties.md new file mode 100644 index 0000000000..9b33af87b0 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_15_duplicate_statvar_properties.md @@ -0,0 +1,15 @@ +# Rule 15: Duplicate Properties in Statistical Variables + +## Description +This rule ensures that no property or concept code is repeated twice within the same Statistical Variable (StatVar) definition. Duplicating dimension values unnecessarily violates the Data Commons structural requirements and can lead to inconsistent behavior downstream. + +## Implementation Details +* **Script:** `scripts/test_rule_15.py` +* **Target:** Parsed `*_stat_vars.mcf` (or `.tmcf`) files located in the `processed_data` directory. +* **Validation Logic:** + 1. Scan each file for StatVar definitions, identifying blocks that start with `Node: dcid:...`. + 2. Track all property keys defined inside that specific block. + 3. If any key is encountered more than once within the same node, flag it as a violation. + +## Remediation +If this rule fails, review the PV map generation logic or custom mapping script. Ensure that dimension-value pairs are properly aggregated and that a single concept/property is only mapped once for a given Statistical Variable. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_16_observation_completeness.md b/scripts/un/un_dataset_validator/rules/rule_16_observation_completeness.md new file mode 100644 index 0000000000..f4a0ecf0a1 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_16_observation_completeness.md @@ -0,0 +1,17 @@ +# Rule 16: Observation Completeness (#input tracking) + +## Description +This rule guarantees that no non-empty observations are improperly dropped during processing. The generated `#input` column tracks the origin of each data point, providing a lineage from the source file. Every valid observation cell in the source data should have a corresponding `#input` reference in the output. + +## Implementation Details +* **Script:** `scripts/test_rule_16.py` +* **Target:** + 1. The output `*_data.csv` files generated in the `processed_data` directory. + 2. The raw input `.csv` files stored in the designated input directory. +* **Validation Logic:** + 1. Parse and extract all `#input` lineage cell references (format `filename:row:col`) from all generated output files. + 2. Scan every raw input file to identify cells containing non-empty observation values (e.g., in `OBS_VALUE` columns). + 3. Ensure that every identified valid input cell maps directly to an extracted `#input` coordinate. + +## Remediation +If this rule fails, it indicates a technical problem with the data pipeline where non-empty points are being unexpectedly omitted. Review the transformation and mapping logic to ensure all valid rows are preserved in the final output. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_17_name_constraints.md b/scripts/un/un_dataset_validator/rules/rule_17_name_constraints.md new file mode 100644 index 0000000000..f0d8e15c7c --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_17_name_constraints.md @@ -0,0 +1,46 @@ +# Rule 17: Name Constraints (Bracket and Code Prohibitions) + +## 1. Rule Description +This rule ensures that the generated `name` properties in the dataset's schemas are properly formatted, human-readable, and descriptive. It combines two core checks targeting the `name` property: +* **Bracket Check (17.1):** A node's `name` must not start with a bracket `[` (ignoring leading whitespace). This ensures that names are not just templates or missing descriptions. +* **Code Concept Check (17.2):** A node's `name` must not contain technical identifier codes, concept codes, or file/side identifiers (such as `CL_`, `DSD_`, `FSP`, `TFT`, or uppercase value codes like `ISCED11_02`). + +## 2. Files Involved +* **Output MCF Files:** All MCF files located in the target dataset's `schema` directory (e.g. `sdg_q1-2026_stat_vars.mcf`, `un_codelist_schema_*.mcf`, etc.). + +## 3. Validation Logic & Flow +1. **Iterate Schema Files:** Loop dynamically through all `*.mcf` files in the target dataset's `schema/` directory. +2. **Parse Nodes:** Monitor node IDs and types (`typeOf`). + * *Exclusion:* If the node is of type `StatVarGroup` (e.g. `dcs:StatVarGroup`), skip name checks since grouping hierarchy titles naturally contain valid taxonomic labels like `(ISIC4 - A)`. +3. **Validate `name` Properties:** + * *Check 17.1 (Bracket Start):* Extract the raw name value (removing potential enclosing quotes). If the first non-whitespace character is `[`, fail the check. + * *Check 17.2 (Technical Codes):* Tokenize the name string. Check if any token matches: + * Explicitly forbidden file-side codes like `FSP`, `TFT`, `DSD`, or `CL`. + * Uppercase technical prefix/identifier formats (e.g., words starting with `CL_` or `DSD_`). + * Uppercase value codes containing underscores (such as `ISCED11_02`, `AGG_ANIMAL_PROD`, etc.). + * *Protect Descriptive Acronyms:* Ensure that common uppercase descriptive words/abbreviations (e.g., `GDP`, `FDI`, `UNCLOS`, `CO2`, `ISIC4`, `MGCI`) do not trigger false positives. + +## 4. Python Implementation Strategy (Draft) + +```python +import os +import glob +import re + +class Rule17Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 17 (Name Constraints)") + schema_files = glob.glob(os.path.join(self.schema_dir, "*.mcf")) + prohibited_codes = {"FSP", "TFT", "DSD", "CL"} + errors = [] + + for schema_file in schema_files: + # Parse nodes line-by-line ... + # 1. Track Node and typeOf + # 2. Extract name value + # 3. Perform Bracket Check: name_val.startswith('[') + # 4. Perform Code Check: search for uppercase codes with underscores or in prohibited_codes + # ... + + return len(errors) == 0 +``` diff --git a/scripts/un/un_dataset_validator/rules/rule_18_mcf_single_dcid_node.md b/scripts/un/un_dataset_validator/rules/rule_18_mcf_single_dcid_node.md new file mode 100644 index 0000000000..cbf76a4713 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_18_mcf_single_dcid_node.md @@ -0,0 +1,42 @@ +# Rule 18: Single-DCID Node Declarations and MCF Comma Prevention + +## 1. Rule Description +This rule ensures that every `Node:` declaration in MCF files defines exactly one valid Data Commons Identifier (DCID) and contains **no commas (`,`)**. Additionally, it verifies that any structural hierarchy properties like `specializationOf:` or `memberOf:` are aligned to reference the corrected consolidated parent DCID (using double-underscore `__` joiners) instead of being split by commas. + +## 2. Files Involved +* **Output MCF / Statvars-Group Files:** All MCF files located in the target dataset's `schema` directory (e.g. `SDG_statvars_groups.mcf`, `sdg_stat_vars.mcf`, etc.). + +## 3. Validation Logic & Flow +1. **Iterate MCF Files:** Loop through all `*.mcf` files in the schema and groups directories. +2. **Verify `Node:` lines:** For every line starting with `Node:`, assert that the character `,` does not exist in the DCID identifier value. Commas indicate multi-DCID nodes which are syntactically invalid and will crash the MCF parser. + * *Correct format:* Use `__` to combine multiple identifiers (e.g., `Node: dcid:A__B__C`). + * *Incorrect format:* Using `,` (e.g., `Node: dcid:A,dcid:B,dcid:C`). +3. **Validate `specializationOf:` and `memberOf:` alignments:** + * If a node has a parent relation referencing consolidated identifiers, ensure they use `__` joiners to match the actual consolidated parent `Node` DCID. + * If left comma-separated (e.g. `specializationOf: dcid:A,dcid:B,dcid:C`), the parser interprets them as three separate references, breaking the inheritance tree if the parent node was corrected to `dcid:A__B__C`. + +## 4. Python Implementation Strategy (Draft) + +```python +import os +import glob + +class Rule18Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 18 (Single-DCID Node Declarations and Comma Prevention)") + mcf_files = glob.glob(os.path.join(self.schema_dir, "*.mcf")) + errors = [] + + for mcf_file in mcf_files: + with open(mcf_file, 'r', encoding='utf-8') as f: + for line_num, line in enumerate(f, 1): + clean_line = line.strip() + if clean_line.startswith("Node:"): + node_val = clean_line.replace("Node:", "").strip() + if ',' in node_val: + err_msg = f"Malformed Node with commas at line {line_num}: {clean_line}" + self.log_error(err_msg) + errors.append(err_msg) + + return len(errors) == 0 +``` diff --git a/scripts/un/un_dataset_validator/rules/rule_1_mapping_validation.md b/scripts/un/un_dataset_validator/rules/rule_1_mapping_validation.md new file mode 100644 index 0000000000..845bdb860a --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_1_mapping_validation.md @@ -0,0 +1,24 @@ +# Rule 1: 1:1 Mapping Validation (PV Maps) + +## Requirement +Ensure all UN concepts and codes have a strict 1:1 mapping with the agency-specific DCP schema. + +## Context & Rules (From Meeting Notes) +- There must be no duplicate properties assigned to a single concept. +- Within the Property-Value (PV) maps, the `event code` column must align perfectly with the `constraint property`. +- For example, if the concept is "age" or "poverty status", the `event code` must be "age" or "poverty status" exactly, maintaining consistency across the file. +- This rule applies strictly to the agency-specific schema rather than the base Data Commons mapping. + +## Implementation Logic +1. **Target Files**: Locate and read PV map files (e.g., `CL_*.csv` or other common PV maps within an agency's schema folder). +2. **Column Validation**: + - Verify the existence of the `event code` and `constraint property` columns. + - For every row, ensure the value in the `event code` column exactly matches the value in the `constraint property` column. +3. **1:1 Mapping Enforcement**: + - Validate that each concept has exactly one unique property assigned to it. + - Flag any duplicate properties assigned to a single concept as a validation error. +4. **Scope**: Apply these checks within the context of the respective agency's schema files. + +## Implementation Deviations +- **Canonical Formatting:** The Python script uses a `to_canonical_format` helper (which strips spaces, underscores, and standardizes case) to match `event code` (UnConcept) to `constraint property` (ConstraintProp), rather than requiring exact string equality. +- **Skipped Concepts:** The core concepts `series`, `geography`, `timeperiod`, and `obsvalue` are explicitly skipped during this 1:1 validation check. diff --git a/scripts/un/un_dataset_validator/rules/rule_2_series_to_populationtype.md b/scripts/un/un_dataset_validator/rules/rule_2_series_to_populationtype.md new file mode 100644 index 0000000000..5712173065 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_2_series_to_populationtype.md @@ -0,0 +1,68 @@ +# Rule 2: SERIES Mapping to `populationType` + +## 1. Rule Description +* **Core Rule (2):** The `series` identifier from the input file must be mapped to the `populationType` property on the generated Statistical Variable nodes in the output `.mcf` file. +* **Sub-rule (2.1):** The series code in the `populationType` must be prefixed with the responsible agency's identifier in the format: `UN__SERIES-`. + +## 2. Files Involved +* **Input Dataset / Context:** The raw input data containing the series information. The series code can typically be derived from the input filename (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv` implies the series `AG_FLS_INDEX`) or from an explicit column within the input CSV. +* **Output MCF File:** The generated StatVars file (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf`). + +## 3. Concrete Example (SDG Dataset) +* **Dataset Context:** `SDG` (derived from the folder name `sdg_q1-2026`). Agency becomes `SDG`. +* **Input Series:** `AG_FLS_INDEX`. +* **Expected Prefix Formation:** `UN_` + `SDG` + `_SERIES-` + `AG_FLS_INDEX` = `UN_SDG_SERIES-AG_FLS_INDEX`. +* **Output Check:** Look at the `.mcf` file. For a node like `Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD`, you must find the exact property line: + ``` + populationType: dcid:UN_SDG_SERIES-AG_FLS_INDEX + ``` + +## 4. Validation Logic & Flow +1. **Determine the Agency Context:** The script should determine the agency (e.g., `SDG`, `ILO`, `UNICEF`) based on the root directory of the dataset being processed (e.g., `/all_data/sdg_q1-2026` -> agency is `SDG`). +2. **Determine the Series Code:** Extract the expected series code for the current file being validated. +3. **Parse the Output MCF:** Read the `output_stat_vars.mcf` file. Group properties by their `Node: dcid:...` definition block. +4. **Validate Nodes:** Iterate through all parsed nodes where `typeOf: dcid:StatisticalVariable`. + * Check if the `populationType` key exists. + * Assert that its value exactly equals `dcid:UN__SERIES-`. + +## 5. Python Implementation Strategy + +```python +import os + +def validate_rule_2(mcf_filepath: str, expected_series_code: str, agency_name: str) -> dict: + """ + Validates Rule 2 & 2.1: SERIES mapped to populationType with agency prefix. + """ + agency_upper = agency_name.upper() + expected_population_type = f"dcid:UN_{agency_upper}_SERIES-{expected_series_code}" + + # Assuming a helper function parse_mcf(filepath) returns a list of dictionaries + # where each dictionary represents a Node and its properties. + mcf_nodes = parse_mcf(mcf_filepath) + + errors = [] + + for node in mcf_nodes: + if node.get('typeOf') == 'dcid:StatisticalVariable': + node_id = node.get('Node') + actual_population_type = node.get('populationType') + + if not actual_population_type: + errors.append(f"Node {node_id} is missing 'populationType'.") + elif actual_population_type != expected_population_type: + errors.append(f"Node {node_id} has incorrect populationType. " + f"Expected '{expected_population_type}', got '{actual_population_type}'.") + + if errors: + return {"status": "FAILED", "errors": errors} + return {"status": "PASSED"} + +# Example Usage: +# validate_rule_2(".../SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf", "AG_FLS_INDEX", "SDG") +``` + +## 6. Edge Cases & Considerations +* **Missing `populationType`:** If the `populationType` is completely absent from a StatVar node, the validation should fail for that specific node. +* **Case Sensitivity:** Ensure the agency name is converted to uppercase (e.g., `sdg` -> `SDG`) before forming the expected DCID string. +* **Non-StatVar Nodes:** Ensure the validation only runs on nodes where `typeOf: dcid:StatisticalVariable`. Other nodes defined in the MCF (if any) should not trigger false positives. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_3_geography.md b/scripts/un/un_dataset_validator/rules/rule_3_geography.md new file mode 100644 index 0000000000..4a23e3f141 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_3_geography.md @@ -0,0 +1,101 @@ +# Rule 3: GEOGRAPHY Mapping to `observationAbout` + +## 1. Rule Description +* **Core Rule (3):** The geographical identifiers found in the input dataset's geography columns must be mapped to the `observationAbout` column in the generated output `.csv` file. This mapping is performed using a central Property-Value (PV) map. +* **Sub-rule (3.1):** Any place in the input geography column that represents a geographical entity at the "Country" level or above (e.g., continents, global regions, Earth) *must* be successfully resolved to a Data Commons identifier (DCID). If it cannot be resolved, this failure must be explicitly recorded. + +## 2. Files Involved +* **Input Dataset:** The raw data containing a geography column (e.g., `REF_AREA` or `geo`). +* **Geography PV Map:** `/all_data/pvmap/un_geography_pvmap.csv`. This file acts as the dictionary, translating UN geographical codes to valid DCIDs for output creation. +* **Event Geography CSV:** `/all_data/un_geography.csv`. According to the implementation meeting notes, this file should be referred to for the full geographical hierarchy to determine if a missing/unresolved geography code represents a country-level or higher entity. +* **Output CSV File:** The generated data file (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`), which must contain the `observationAbout` column. + +## 3. Concrete Example (SDG Dataset) +* **Input Dataset:** A row in the input CSV has a geography code: `G00000020`. +* **PV Map Lookup:** The script looks up `GEOGRAPHY:G00000020` in `un_geography_pvmap.csv` and finds the mapping to the DCID `dcid:country/AFG`. +* **Output Check:** In the corresponding row of the generated `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`, the `observationAbout` column must contain the value `dcid:country/AFG`. + +## 4. Validation Logic & Flow +1. **Load the References:** + * Read `un_geography_pvmap.csv` into a Python dictionary to know the expected resolved DCIDs. + * Read `un_geography.csv` into a dictionary/dataframe to understand the geography names and their hierarchy levels (to identify if a code is a country, continent, etc.). +2. **Identify Target Columns:** + * Locate the geography column in the input CSV. + * Locate the `observationAbout` column in the output CSV. +3. **Iterate and Compare:** + * Read the geography code from the input row. + * Format it to match the PV map key (e.g., prepend `GEOGRAPHY:`). + * Lookup the expected DCID in the PV map. + * Verify that the actual value in the output CSV's `observationAbout` column matches the expected DCID. +4. **Handle Unresolved Geographies (Rule 3.1):** + * If a geography code is *not* found in the PV map (or maps to an `#ignore` / unmapped place), it is unresolved. + * Look up this unresolved code in the `un_geography.csv` file. + * If the entity represents a country-level or higher place (e.g., 'Earth', 'Asia', 'Country'), record the code and its name to an explicit error log (e.g., `missing_national_geographies.log`). + +## 5. Python Implementation Strategy + +```python +import pandas as pd + +def validate_rule_3(input_csv_path: str, output_csv_path: str, pv_map_path: str, un_geo_csv_path: str, geo_input_col_name: str = 'geo') -> dict: + """ + Validates Rule 3 & 3.1: Geography mapping and logging unresolved national/higher entities. + """ + # 1. Load PV Map + pv_df = pd.read_csv(pv_map_path, comment='#', names=['GeoCode', 'Prop', 'Value']) + geo_map = dict(zip(pv_df['GeoCode'].astype(str), pv_df['Value'].astype(str))) + + # 2. Load Event Geography CSV (Hierarchy Info) + # Allows us to check if an unresolved code is Country level or above + un_geo_df = pd.read_csv(un_geo_csv_path) + # Creating a dictionary for fast lookup: Code -> Name + un_geo_names = dict(zip(un_geo_df['CODE'].astype(str), un_geo_df['NAME_EN'].astype(str))) + + # 3. Load Input and Output data + input_df = pd.read_csv(input_csv_path) + output_df = pd.read_csv(output_csv_path) + + if geo_input_col_name not in input_df.columns: + return {"status": "FAILED", "errors": [f"Input column '{geo_input_col_name}' not found."]} + if 'observationAbout' not in output_df.columns: + return {"status": "FAILED", "errors": ["Output column 'observationAbout' not found."]} + + errors = [] + unresolved_high_level_geos = set() + + # Using the #input tracker is highly recommended to safely join the data, + # but for simplicity, assuming a row-by-row mapping here: + for idx in range(min(len(input_df), len(output_df))): + raw_code = str(input_df[geo_input_col_name].iloc[idx]).strip() + pv_lookup_code = f"GEOGRAPHY:{raw_code}" + + actual_output = str(output_df['observationAbout'].iloc[idx]).strip() + expected_output = geo_map.get(pv_lookup_code) + + # Rule 3.1: Capture missing national or higher places + is_unresolved = (not expected_output) or (expected_output == 'unmapped place') + + if is_unresolved: + geo_name = un_geo_names.get(raw_code, "Unknown Name") + + # Placeholder check for "country level or above". + # In production, this would use a more robust check based on hierarchy / dc_parent_id in un_geography.csv + # For now, flag it for recording. + unresolved_high_level_geos.add(f"{raw_code} ({geo_name})") + + elif actual_output != expected_output: + errors.append(f"Row {idx}: Geo mismatch. Input '{raw_code}' -> Expected '{expected_output}', Got '{actual_output}'") + + response = {"status": "PASSED" if not errors else "FAILED"} + if errors: + response["errors"] = errors + if unresolved_high_level_geos: + # Rule 3.1 mandates recording these + response["unresolved_high_level_places"] = list(unresolved_high_level_geos) + + return response +``` + +## 6. Edge Cases & Considerations +* **Hierarchy Resolution:** `un_geography.csv` might not have an explicit "level" column (like "country", "continent"). The script might need to infer level based on DCID structure (e.g., `country/XXX` vs `Earth`) or by resolving parents recursively using the `PARENT` column until it reaches a known global root. +* **Row Alignment with `#input`:** Instead of relying on index matching (`idx`), it is much safer to join the generated `SDG_..._data.csv` to the source input CSV using the `#input` column present in the output file, avoiding mismatch errors caused by dropped rows. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_4_time_period.md b/scripts/un/un_dataset_validator/rules/rule_4_time_period.md new file mode 100644 index 0000000000..56ec3ee385 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_4_time_period.md @@ -0,0 +1,82 @@ +# Rule 4: Time Period to observationDate + +## Objective +Ensure that the `TIME_PERIOD` (or `timePeriod`) column in the source data accurately maps to the Data Commons `observationDate` property in the output CSV, adhering to the new conversion requirements for complex formats. + +## File References +- **Input Data File:** Source data file containing the `TIME_PERIOD` column. +- **Output CSV:** Transcoded data file (e.g., `processed_data/*_data.csv`). + +## Validation Logic & Flow +1. **Extraction:** Read the `TIME_PERIOD` column from the input source data. +2. **Target Resolution:** Read the corresponding `observationDate` column in the generated output CSV via `#input` linkage. +3. **Verification Checks:** + - **Conversion Check:** The data pipeline converts non-standard date formats into standard formats. Check if `observationDate` equals the first 4 characters (YYYY) of the `TIME_PERIOD` string. + - **Interval Start Date Extraction:** If the input time format is an ISO-8601 interval (e.g., `2014-01/P3M`), the data pipeline accurately extracts the start date component (`2014-01`). Check if `observationDate` equals this start date. + - **Fallback Check:** If the conversion checks fail, check if `observationDate` exactly matches the full `TIME_PERIOD` string. This ensures that unconvertible formats are passed through unaltered. + +## Python Implementation Strategy +```python +import pandas as pd + +def validate_time_period(input_csv_path, output_csv_path): + # Load data + input_df = pd.read_csv(input_csv_path) + output_df = pd.read_csv(output_csv_path) + + target_col = 'observationDate' + source_col = 'timePeriod' + + if target_col not in output_df.columns: + print(f"ERROR: Target column '{target_col}' not found in output CSV.") + return False + + if source_col not in input_df.columns: + source_cols_upper = {c.upper(): c for c in input_df.columns} + if 'TIME_PERIOD' in source_cols_upper: + source_col = source_cols_upper['TIME_PERIOD'] + else: + print(f"ERROR: Source column representing Time Period not found in input CSV.") + return False + + success = True + for idx, row in output_df.iterrows(): + # Get input row via #input logic + # For simplicity in this pseudocode, assuming 1:1 same row index + input_time = str(input_df.loc[idx, source_col]).strip() + actual_obs_date = str(row[target_col]).strip() + + # 1. YYYY format check + yyyy_format = input_time[:4] + + if actual_obs_date == yyyy_format: + continue + + # 2. Straight string equivalence check + if actual_obs_date == input_time: + continue + + # 3. Interval start date check + if '/' in input_time: + start_date = input_time.split('/')[0] + if actual_obs_date == start_date: + continue + + print(f"FAILED: Time Period mismatch. Expected '{yyyy_format}', '{start_date}', or '{input_time}', Found: '{actual_obs_date}'") + success = False + + return success +``` + +## Example Scenario +- **Input `timePeriod`:** `2015` +- **Output CSV Column (`observationDate`):** `2015` +- **Validation Result:** Pass + +- **Input `timePeriod`:** `2024-25/P3M` +- **Output CSV Column (`observationDate`):** `2024-25` +- **Validation Result:** Pass (successfully extracted interval start date). + +- **Input `timePeriod`:** `ComplexData` +- **Output CSV Column (`observationDate`):** `ComplexData` +- **Validation Result:** Pass (fallback to string equivalence). diff --git a/scripts/un/un_dataset_validator/rules/rule_5_obs_value_to_value.md b/scripts/un/un_dataset_validator/rules/rule_5_obs_value_to_value.md new file mode 100644 index 0000000000..7fb00a6ee6 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_5_obs_value_to_value.md @@ -0,0 +1,70 @@ +# Rule 5: OBS_VALUE Mapping to `value` + +## 1. Rule Description +* **Core Rule (5):** The observation value (`OBS_VALUE` column) from the input data must be mapped to the `value` property in the output CSV. +* **Format Check:** As per the meeting notes, observation values are required to map to `value.number`. This means the resulting `value` must be a valid number (`{Number}`). This validation check must be performed using the property-value (PV) map. + +## 2. Files Involved +* **Input Dataset:** The raw input `.csv` data files containing the `OBS_VALUE` column. +* **Output Data File:** The generated output `.csv` file. +* **Property Value Map (PV Map):** Specifically, the `common_pvmap_obs.csv` file which defines the mapping rule: `OBS_VALUE,value,{Number},...` + +## 3. Concrete Example +* **Input Data Row:** Contains an `OBS_VALUE` of `594982.4482`. +* **Output Check:** The output `.csv` file should contain a column named `value`, and the corresponding row must contain `594982.4482` (or potentially a multiplied value if unit multipliers apply, see Rule 10, but inherently it must be numeric). + +## 4. Validation Logic & Flow +1. **Identify the Input and Output Files:** Pair the input dataset `.csv` file with its corresponding generated output `.csv` file. +2. **Column Verification:** + * Verify that the input file has an `OBS_VALUE` column. + * Verify that the output file has a `value` column. +3. **Row-by-Row Mapping Validation:** + * For each row in the input, locate the matching row in the output (this may require a row identifier like `#input`). + * **Value Mapping Check:** The `OBS_VALUE` should be correctly transferred to the `value` column. + * **Number Validation:** Ensure that the data in the output `value` column can be parsed as a float/number. + +## 5. Python Implementation Strategy + +```python +import pandas as pd + +def validate_rule_5(input_csv_path: str, output_csv_path: str) -> dict: + """ + Validates Rule 5: OBS_VALUE maps to value and is numeric. + """ + errors = [] + + try: + input_df = pd.read_csv(input_csv_path) + output_df = pd.read_csv(output_csv_path) + except Exception as e: + return {"status": "FAILED", "errors": [f"File read error: {e}"]} + + if 'OBS_VALUE' not in input_df.columns: + # Assuming all valid input files must have OBS_VALUE for this rule to apply + return {"status": "SKIPPED", "message": "No OBS_VALUE column in input."} + + if 'value' not in output_df.columns: + errors.append("Output CSV is missing the 'value' column.") + return {"status": "FAILED", "errors": errors} + + # Verify that all entries in the output 'value' column are numeric + # We use pd.to_numeric with errors='coerce' which turns non-numeric into NaN + numeric_values = pd.to_numeric(output_df['value'], errors='coerce') + + # Check for rows where the value became NaN (but wasn't originally empty) + non_numeric_mask = numeric_values.isna() & output_df['value'].notna() + + if non_numeric_mask.any(): + bad_rows = output_df[non_numeric_mask].index.tolist() + errors.append(f"Found non-numeric values in the 'value' column at output row indices: {bad_rows[:5]}...") + + if errors: + return {"status": "FAILED", "errors": errors} + return {"status": "PASSED"} +``` + +## 6. Edge Cases & Considerations +* **Empty/Missing Values:** How should missing or empty `OBS_VALUE` entries be handled? They should likely map to empty/missing in the output `value` and not fail the numeric check. +* **Multipliers (Rule 10 Interaction):** If a unit multiplier is present, the final `value` in the output will be `OBS_VALUE * MULTIPLIER`. The validation script might need to account for this multiplication rather than expecting an exact string match. However, the requirement that the resulting `value` is `{Number}` remains absolute. +* **Special Characters:** Check for formatted numbers in the input (e.g., `"1,000.50"`). These must be clean numeric values (e.g., `1000.50`) in the output. diff --git a/scripts/un/un_dataset_validator/rules/rule_6_1_dimension_mapping_prefix.md b/scripts/un/un_dataset_validator/rules/rule_6_1_dimension_mapping_prefix.md new file mode 100644 index 0000000000..442cc83274 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_6_1_dimension_mapping_prefix.md @@ -0,0 +1,20 @@ +# Rule 6.1: Dimension Mapping to DCP Schema with UN_ Prefix + +## Requirement +Each dimension must be mapped to the Data Commons Project (DCP) schema using a common prefix `UN_` without including the specific agency prefix. + +## Context & Rules (From Meeting Notes) +- The dataset is processed to create a schema. Any schema generated from the DCP must consistently use the `UN_` prefix for dimensions. +- Every node within the agency-specific schema files follows a "concept_value" structure. +- The mapping must ensure these dimensions correctly represent the concept without introducing agency-specific identifiers in the prefix (e.g., use `UN_` instead of `UN_ECLAC_`). + +## Implementation Logic +1. **Target Files**: Locate and read the generated schema files or the mapping configurations that define how dimensions are output. +2. **Prefix Validation**: + - For every dimension identified (where the ROLE is 'dimension' in the respective DSD), verify the resulting mapped output. + - Check that the prefix applied is strictly `UN_`. + - Flag an error if the prefix contains an agency name (e.g., `UN_ILO_` or `UN_WHO_`) or if the `UN_` prefix is entirely missing. +3. **Scope**: This applies to all datasets processed to create the DCP schema across all agencies. + +## Implementation Deviations +- **Temporarily Excluded:** During the June 23rd meeting, it was decided that all validations mapping strictly to the base DCP schema (such as enforcing the `UN_` prefix mapping) are temporarily ignored pending further clarification from AJ. The validation script currently skips this check. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_6_2_dimension_value_mapping.md b/scripts/un/un_dataset_validator/rules/rule_6_2_dimension_value_mapping.md new file mode 100644 index 0000000000..bc7b343601 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_6_2_dimension_value_mapping.md @@ -0,0 +1,96 @@ +# Rule 6.2: Dimension Value Mapping + +## Objective +Ensure that every value within a dimension column is properly mapped to a generated property value DCID following the exact template: `_-`. + +## Context & Rationale +According to the meeting notes and project requirements, any column designated as a "dimension" in the Dataset Definition (DSD) file must have its distinct values transformed into standardized Data Commons Identifiers (DCIDs). The format for these DCIDs consistently incorporates a prefix (typically `UN`), the concept name (derived from the column name), and the specific value code. + +For instance, if the dimension column is `product` and a raw value in the dataset is `CPC2_1_0113`, the mapped value DCID should be formatted as `UN_PRODUCT-CPC2_1_0113`. In the output schema/MCF, this would appear as a property-value pair like `product: dcid:UN_PRODUCT-CPC2_1_0113`. + +## Files Involved +* **DSD File (e.g., `schema.csv` or `DSD.csv`):** Used to identify which columns act as dimensions (where `ROLE` = `Dimension`). +* **Input Data File (`.csv`):** Provides the raw source values for the identified dimension columns. +* **Output Files (`.mcf`, `.tmcf`, or processed `.csv`):** Validated to ensure the final output respects the mapped DCID format. + +## Verification Logic +1. **Identify Dimensions:** Read the DSD file and filter for rows where the `ROLE` column is explicitly set to `Dimension` (excluding explicit exceptions like Geography, Time Period, OBS_VALUE, and Series, which have their own rules). +2. **Extract Concept Names:** Determine the concept name from the dimension's column name. The concept name is generally the uppercase version of the column name (e.g., `product` -> `PRODUCT`). +3. **Construct Expected DCIDs:** For every row in the input data file, look at the value under the dimension column. Construct the expected DCID using the template: + * `Prefix`: `UN` (or derived from the specific dataset configuration). + * `Concept`: Capitalized column name. + * `Code`: The raw value found in the input data. + * **Format:** `_-` (e.g., `UN_PRODUCT-CPC2_1_0113`). +4. **Validate Output:** Ensure that in the finalized statistical variable mapping or output nodes, the dimension property maps exactly to this constructed DCID (e.g., checking that `product: dcid:UN_PRODUCT-CPC2_1_0113` exists). + +## Python Implementation Strategy + +```python +import pandas as pd +import re + +def validate_dimension_value_mapping(dsd_path: str, data_csv_path: str, output_mcf_path: str, prefix: str = "UN"): + """ + Validates that values for dimension columns are mapped to the correct DCID template. + """ + # 1. Read DSD and identify dimension columns + dsd_df = pd.read_csv(dsd_path) + + # Identify dimensions but exclude Geography, Time Period, OBS_VALUE, and SERIES as per rules + exclusions = ['GEOGRAPHY', 'TIME PERIOD', 'OBS_VALUE', 'TIME_PERIOD', 'SERIES'] + dimensions = dsd_df[ + (dsd_df['ROLE'].str.upper() == 'DIMENSION') & + (~dsd_df['COLUMN_NAME'].str.upper().isin(exclusions)) + ]['COLUMN_NAME'].tolist() + + # 2. Read input data to collect unique values for each dimension + data_df = pd.read_csv(data_csv_path) + + expected_mappings = {} + for dim in dimensions: + if dim in data_df.columns: + concept = dim.upper() + unique_values = data_df[dim].dropna().unique() + expected_dcids = [] + for val in unique_values: + # Convert special characters to underscores (as per notes) + clean_val = re.sub(r'[^a-zA-Z0-9_]', '_', str(val)) + expected_dcids.append(f"{prefix}_{concept}-{clean_val}") + + expected_mappings[dim] = expected_dcids + + # 3. Read output MCF/TMCF file content to verify + with open(output_mcf_path, 'r') as f: + mcf_content = f.read() + + validation_errors = [] + + # 4. Verify that the constructed DCIDs exist in the output mapping + for dim, expected_dcids in expected_mappings.items(): + for expected_dcid in expected_dcids: + # Check for property: dcid: format + # e.g., product: dcid:UN_PRODUCT-CPC2_1_0113 + # NOTE: Series is a dimension but maps to populationType, which may need a custom check. + expected_string = f"{dim}: dcid:{expected_dcid}" + + if expected_string not in mcf_content: + # Also fallback to check just the DCID presence in case property name differs + if expected_dcid not in mcf_content: + validation_errors.append( + f"Missing or incorrectly formatted dimension mapping. " + f"Expected to find: {expected_dcid} for dimension '{dim}'." + ) + + return validation_errors + +# Example usage: +# errors = validate_dimension_value_mapping('schema.csv', 'data.csv', 'output.mcf') +# if errors: +# for e in errors: +# print(e) +``` + +## Edge Cases to Consider +* **Special Characters:** If the raw value (``) contains special characters or spaces, they must be converted to underscores before constructing the DCID (as per the separate special character conversion rule). +* **Case Sensitivity:** Ensure that the prefix and concept are properly cased (e.g., uppercase `UN` and `PRODUCT`) while the value code's case might depend on specific entity resolution rules (typically preserved or uppercase). +* **Missing Values:** If a dimension column has a missing/null value for a specific row, the validation script should safely ignore it or flag it as an error based on strictness requirements, rather than generating an invalid DCID like `UN_PRODUCT-nan`. diff --git a/scripts/un/un_dataset_validator/rules/rule_6_dimensions_and_attributes.md b/scripts/un/un_dataset_validator/rules/rule_6_dimensions_and_attributes.md new file mode 100644 index 0000000000..e610f2a766 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_6_dimensions_and_attributes.md @@ -0,0 +1,96 @@ +# Rule 6: Dimensions vs. Attributes + +## 1. Rule Description +* **Core Rule (6):** All properties defined in the Dataset Definition (DSD) file must be correctly handled as either a **dimension** or an **attribute**. +* **Dimensions:** Except for specific exemptions (`Geography`, `Time Period`, `OBS_VALUE`, and `SERIES`), any property marked as a "dimension" must be attached to the Statistical Variable (StatVar) as a **constraint property**. +* **Attributes:** Any property marked as an "attribute" must **not** be attached to the Statistical Variable. Instead, it must be included in the output data `.csv` as a separate column. + +## 2. Files Involved +* **Input DSD File:** The respective DSD file for the dataset (e.g., an Excel/CSV file containing the `ROLE` column with values "dimension" or "attribute"). +* **Input Data File:** The raw `.csv` data. +* **Output Data File:** The generated output `.csv` file. +* **Output MCF File:** The generated `.mcf` file containing the Statistical Variable definitions. + +## 3. Concrete Example +* **DSD Definition:** + * `PRODUCT` is defined with a `ROLE` of "dimension". + * `CENSORED_VALUE_TYPE` is defined with a `ROLE` of "attribute". +* **Output Check for Dimension (`PRODUCT`):** + * The Statistical Variable node (e.g., `Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_CRL_PUL`) in the `.mcf` file must contain `PRODUCT` as a constraint property (e.g., `product: dcid:UN_PRODUCT-AGG_CRL_PUL`). +* **Output Check for Attribute (`CENSORED_VALUE_TYPE`):** + * The StatVar node must **not** contain `CENSORED_VALUE_TYPE`. + * The output `.csv` file must contain a separate column named `censoredValueType` (or similar mapped name), with values like `UN_CENSORED_VALUE_TYPE-_Z`. + +## 4. Validation Logic & Flow +1. **Parse DSD File:** Read the DSD file and extract the list of dimensions and attributes from the `ROLE` column. +2. **Filter Dimensions:** From the list of dimensions, exclude the predefined exemptions: Geography (e.g., `REF_AREA`), Time Period (e.g., `TIME_PERIOD`), and Observation Value (`OBS_VALUE`). +3. **Validate Dimensions in MCF:** + * Iterate through the generated StatVars in the output `.mcf` file. + * Verify that the remaining dimensions (e.g., `SEX`, `AGE`, `PRODUCT`) exist as constraint properties attached to the respective StatVar nodes. +4. **Validate Attributes in CSV:** + * Iterate through the list of attributes defined in the DSD. + * Verify that they do **not** appear in the StatVar nodes. + * Verify that they **do** appear as distinct columns in the generated output `.csv` file. + +## 5. Python Implementation Strategy + +```python +import pandas as pd +import re + +def validate_rule_6(dsd_csv_path: str, output_csv_path: str, output_mcf_path: str) -> dict: + """ + Validates Rule 6: Dimensions as constraint properties, Attributes as separate columns. + """ + errors = [] + + try: + dsd_df = pd.read_csv(dsd_csv_path) + output_df = pd.read_csv(output_csv_path) + with open(output_mcf_path, 'r') as f: + mcf_content = f.read() + except Exception as e: + return {"status": "FAILED", "errors": [f"File read error: {e}"]} + + if 'ROLE' not in dsd_df.columns or 'concept' not in dsd_df.columns: + return {"status": "SKIPPED", "message": "DSD file missing 'ROLE' or 'concept' columns."} + + # Identify Dimensions and Attributes (ignoring case variations if any) + dimensions = dsd_df[dsd_df['ROLE'].str.lower() == 'dimension']['concept'].tolist() + attributes = dsd_df[dsd_df['ROLE'].str.lower() == 'attribute']['concept'].tolist() + + # Exemptions + exemptions = ['REF_AREA', 'TIME_PERIOD', 'OBS_VALUE', 'SERIES'] + target_dimensions = [dim for dim in dimensions if dim.upper() not in [e.upper() for e in exemptions]] + + # Validate Attributes in CSV + # Attribute names might be mapped (e.g., CENSORED_VALUE_TYPE -> censoredValueType) + # This check might need fuzzy matching or a mapping dictionary if names differ. + for attr in attributes: + # Simplistic check: assumes attribute name in DSD maps to a column name or a variation + # In practice, you might need a PV map to resolve the exact CSV column name. + match_found = any(attr.lower() in col.lower().replace("_", "") for col in output_df.columns) + if not match_found: + errors.append(f"Attribute '{attr}' not found as a column in the output CSV.") + + # Also ensure it is NOT in the MCF as a property + if re.search(rf'^{attr.lower()}:', mcf_content, re.IGNORECASE | re.MULTILINE): + errors.append(f"Attribute '{attr}' incorrectly attached to a StatVar in the MCF file.") + + # Validate Dimensions in MCF + for dim in target_dimensions: + # Check if the dimension appears as a constraint property in the MCF + if not re.search(rf'^{dim.lower()}:', mcf_content, re.IGNORECASE | re.MULTILINE): + # Note: This is a file-level check. A strict row-by-row mapping might be required + # if only specific StatVars should have specific dimensions. + errors.append(f"Dimension '{dim}' not found as a constraint property in the MCF file.") + + if errors: + return {"status": "FAILED", "errors": errors} + return {"status": "PASSED"} +``` + +## 6. Edge Cases & Considerations +* **Exemptions List:** The list of dimensions to exempt (`Geography`, `Time Period`, and `OBS_VALUE`) might use different column names in different datasets (e.g., `REF_AREA` vs `geography`). The validation script must map these correctly based on the dataset. +* **Naming Conventions:** DSD concept names (e.g., `CENSORED_VALUE_TYPE`) might be camelCased or otherwise transformed when they become output CSV column names (e.g., `censoredValueType`). The validation logic needs to account for this transformation. +* **Missing DSD Columns:** If a DSD file is missing the `ROLE` column, the script needs a fallback or should flag it as an immediate error, as the distinction cannot be made. diff --git a/scripts/un/un_dataset_validator/rules/rule_7_unit_multiplier.md b/scripts/un/un_dataset_validator/rules/rule_7_unit_multiplier.md new file mode 100644 index 0000000000..b7dab91b69 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_7_unit_multiplier.md @@ -0,0 +1,84 @@ +# Rule 7: UNIT_MULTIPLIER Application + +## Mapping Logic +1. **Identify Multiplier Attribute:** Identify the column representing the unit multiplier in the raw input data (often named `UNIT_MULT`, `UNIT_MULTIPLIER`, etc., depending on the DSD). +2. **Lookup Factor:** Use the common PV map `all_data/pvmap/CL_MULT_pvmap_multiply.csv` to resolve the multiplication factor. The PV map links the multiplier code to a float value (e.g., `1.00E-15` for -15). +3. **Apply Factor:** The expected value in the Data Commons output CSV should be: + `Expected_Value = float(Raw_OBS_VALUE) * float(Multiplier_Factor)` +4. **Compare:** Assert that `Expected_Value` matches the `value` column in the processed output CSV. + +## Relevant Files +- **Raw Input Data:** Varies per dataset (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv`). +- **PV Map for Multipliers:** `all_data/pvmap/CL_MULT_pvmap_multiply.csv`. +- **Processed Output Data:** Varies per dataset (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv` in `processed_data/`). + +## Python Implementation Strategy + +```python +import pandas as pd +import numpy as np + +def validate_unit_multiplier(input_csv_path: str, output_csv_path: str, dsd_path: str, pvmap_path: str = "all_data/pvmap/CL_MULT_pvmap_multiply.csv"): + """ + Validates that the UNIT_MULTIPLIER factor is applied to the output values. + """ + # 1. Load the PV map for multipliers + # Format: UnCodeKey,prop,val -> e.g., UNIT_MULT:-15,#Multiply,1.00E-15 + multiplier_map = {} + pv_df = pd.read_csv(pvmap_path) + for _, row in pv_df.iterrows(): + key = str(row['UnCodeKey']) # e.g., 'UNIT_MULT:-15' + val = float(row['val']) # e.g., 1e-15 + multiplier_map[key] = val + + # 2. Identify the multiplier column from DSD + dsd_df = pd.read_csv(dsd_path) + + # Check if there is a multiplier attribute + multiplier_cols = dsd_df[dsd_df['COLUMN_NAME'].str.upper().str.contains('MULT|MULTIPLIER')]['COLUMN_NAME'].tolist() + + if not multiplier_cols: + return [] # No multiplier column in this dataset + + mult_col = multiplier_cols[0] + + # 3. Read input and output data + input_df = pd.read_csv(input_csv_path) + output_df = pd.read_csv(output_csv_path) + + # Ensure order aligns for row-by-row comparison, or merge on keys + # Assuming direct row equivalence for simplicity, but ideally merge on dimensions + if len(input_df) != len(output_df): + return ["Row count mismatch between input and output CSV."] + + validation_errors = [] + + # 4. Validate multiplication + for idx, (in_row, out_row) in enumerate(zip(input_df.iterrows(), output_df.iterrows())): + _, input_data = in_row + _, output_data = out_row + + if pd.isna(input_data[mult_col]): + continue # No multiplier for this row + + mult_val = str(input_data[mult_col]) + # Construct the key expected in the PV Map + # Note: Depending on data, this might just be mult_col + ":" + mult_val + pv_key = f"{mult_col.upper()}:{mult_val}" + + if pv_key in multiplier_map: + factor = multiplier_map[pv_key] + + raw_val = float(input_data['OBS_VALUE']) if 'OBS_VALUE' in input_data else float(input_data.get('value', 0)) + expected_val = raw_val * factor + + actual_val = float(output_data['value']) + + # Allow minor floating point drift + if not np.isclose(expected_val, actual_val, rtol=1e-5): + validation_errors.append(f"Row {idx}: Multiplier validation failed. Raw: {raw_val}, Factor: {factor}, Expected: {expected_val}, Actual: {actual_val}") + else: + validation_errors.append(f"Row {idx}: Multiplier key '{pv_key}' not found in PV map.") + + return validation_errors +``` \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_8_unit_measure_mapping.md b/scripts/un/un_dataset_validator/rules/rule_8_unit_measure_mapping.md new file mode 100644 index 0000000000..c503b67678 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_8_unit_measure_mapping.md @@ -0,0 +1,19 @@ +# Rule 8: UNIT_MEASURE Mapping and Multiplier Logic Integration + +## Requirement +`UNIT_MEASURE` must be mapped to a Data Commons Project (DCP) enum using the name from the source dataset. Additionally, validation logic for `UNIT_MEASURE` should be closely integrated with multiplier logic. + +## Context & Rules (From Meeting Notes) +- The checklist initially marked this as an "ASK AJAI" item regarding how to map `UNIT_MEASURE` to a DCP enum and what to set in the `shortDisplayName`. +- During the meeting, it was agreed that the mapping for rule number eight involves similar multiplier logic to what is used for applying multipliers (Rule 7). +- The decision was finalized to integrate this mapping and the associated multiplier logic into the existing validation code to ensure consistency when validating how units and multipliers are applied to values. + +## Implementation Logic +1. **Target Files**: Source data files (containing `UNIT_MEASURE`), DSD files, and schema mapping files. +2. **Enum Mapping Validation**: + - Verify that each `UNIT_MEASURE` from the source is correctly mapped to an existing DCP enum. + - The validation should check that the name used for the enum corresponds to the unit defined in the source data. +3. **Multiplier Logic Integration**: + - Tie the validation of `UNIT_MEASURE` with the validation of `UNIT_MULTIPLIER`. + - Ensure the code checks that when a unit is applied, any corresponding multiplier logic (e.g., scaling the value based on the multiplier enum) is also evaluated correctly. + - Flag any mismatches where a unit measure does not successfully map to a DCP enum or where the integrated multiplier application fails. \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/rules/rule_9_frequency_to_observationperiod.md b/scripts/un/un_dataset_validator/rules/rule_9_frequency_to_observationperiod.md new file mode 100644 index 0000000000..7b537be089 --- /dev/null +++ b/scripts/un/un_dataset_validator/rules/rule_9_frequency_to_observationperiod.md @@ -0,0 +1,67 @@ +# Rule 9: Frequency to observationPeriod + +## Objective +Ensure that the `FREQUENCY` column in the source data correctly maps to the Data Commons `observationPeriod` property in the output CSV, adhering to the mapping defined in the common Property-Value (PV) map. + +## File References +- **Input Data File:** Data file containing the `FREQUENCY` column. +- **Common PV Map:** `all_data/pvmap/CL_FREQUENCY_pvmap_obsperiod.csv` +- **Output CSV:** Transcoded data file (e.g., `processed_data/*_data.csv`). + +## Validation Logic & Flow +1. **Extraction:** Read the `FREQUENCY` column from the input source data. +2. **Lookup:** Cross-reference the input frequency code (e.g., `A`, `M`, `Q`) with the `UnCode` column in `CL_FREQUENCY_pvmap_obsperiod.csv`. +3. **Target Resolution:** Extract the mapped Data Commons observation period from the `ConstraintPropValue` column in the PV map (e.g., `P1Y` for Annual). +4. **Verification:** Check the corresponding row in the output CSV and ensure the resolved value is present in the `observationPeriod` column. + +## Python Implementation Strategy +```python +import pandas as pd + +def validate_frequency(input_csv_path, output_csv_path, pv_map_path): + # Load data + input_df = pd.read_csv(input_csv_path) + output_df = pd.read_csv(output_csv_path) + pv_map_df = pd.read_csv(pv_map_path) + + # Anomaly Handling: Check for typos like 'opservationPeriod' in output headers + target_col = 'observationPeriod' + if target_col not in output_df.columns: + if 'opservationPeriod' in output_df.columns: + print("ERROR: Typo found in output CSV header: 'opservationPeriod' instead of 'observationPeriod'.") + # For validation continuity, we can temporarily map it, but it should be flagged as a failure. + target_col = 'opservationPeriod' + else: + print(f"ERROR: Target column '{target_col}' not found in output CSV.") + return False + + # Create mapping dictionary from PV Map + # Note: Sometimes the UnCode has quotes, so we strip them + pv_map_df['UnCode'] = pv_map_df['UnCode'].str.strip('"') + freq_map = dict(zip(pv_map_df['UnCode'], pv_map_df['ConstraintPropValue'])) + + success = True + for idx, row in input_df.iterrows(): + input_freq = str(row.get('FREQUENCY', '')).strip() + expected_obs_period = freq_map.get(input_freq) + + if not expected_obs_period: + print(f"WARNING: Unmapped FREQUENCY code '{input_freq}' at row {idx}") + continue + + actual_obs_period = str(output_df.loc[idx, target_col]).strip() + + if actual_obs_period != expected_obs_period: + print(f"FAILED: Mismatch at row {idx}. Expected: {expected_obs_period}, Found: {actual_obs_period}") + success = False + + return success +``` + +## Important Anomaly Handling +As noted in the implementation analysis, the output pipeline has been known to generate files with the header `opservationPeriod` instead of `observationPeriod`. The Python script must dynamically check for this typo and flag the validation as a failure if the correct header is missing, reporting the specific typo found. + +## Example Scenario +- **Input `FREQUENCY` code:** `A` +- **PV Map Lookup (`UnCode` -> `ConstraintPropValue`):** `A` -> `P1Y` +- **Output CSV Column (`observationPeriod`):** `P1Y` diff --git a/scripts/un/un_dataset_validator/run_custom_dataset.sh b/scripts/un/un_dataset_validator/run_custom_dataset.sh new file mode 100755 index 0000000000..6b198462ce --- /dev/null +++ b/scripts/un/un_dataset_validator/run_custom_dataset.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Simple wrapper script to run validations on a custom dataset + +if [ "$#" -ne 4 ]; then + echo "Usage: ./run_custom_dataset.sh " + echo "Example: ./run_custom_dataset.sh my_dataset ../my_data_folder/my_dataset ../raw_data/DATA ../raw_data/DSD" + exit 1 +fi + +DATASET_NAME="$1" +PROCESSED_DIR="$2" +INPUT_DATA_DIR="$3" +DSD_DIR="$4" +LOG_DIR="logs/${DATASET_NAME}_validation_logs" + +echo "=======================================================" +echo "Running Validation Suite for: $DATASET_NAME" +echo "Processed Directory: $PROCESSED_DIR" +echo "Input Data: $INPUT_DATA_DIR" +echo "DSD Directory: $DSD_DIR" +echo "Logs will be saved to: un_dataset_validator/$LOG_DIR" +echo "=======================================================" + +# Ensure we're in the validation directory relative to where the script is run +SCRIPT_DIR=$(dirname "$0") +cd "$SCRIPT_DIR" || exit 1 + +# Create log directory if it doesn't exist +mkdir -p "$LOG_DIR" + +# Run the python validation suite +python3 scripts/run_validations.py --dataset "$DATASET_NAME" --dataset_dir "$PROCESSED_DIR" --input_data_dir "$INPUT_DATA_DIR" --dsd_dir "$DSD_DIR" --rule all --log_dir "$LOG_DIR" + +if [ $? -eq 0 ]; then + echo "Validation completed successfully!" +else + echo "Validation finished with errors. Please review the logs." +fi + +echo "Summary report generated at: un_dataset_validator/$LOG_DIR/summary.md" diff --git a/scripts/un/un_dataset_validator/scripts/base_validator.py b/scripts/un/un_dataset_validator/scripts/base_validator.py new file mode 100644 index 0000000000..a40bed8997 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/base_validator.py @@ -0,0 +1,37 @@ +import os +from abc import ABC, abstractmethod +from datetime import datetime + +class BaseRuleValidator(ABC): + def __init__(self, dataset_name: str, dataset_dir: str, input_data_dir: str = None, dsd_dir: str = None, log_dir: str = "un_dataset_validator/logs"): + self.dataset_name = dataset_name + self.dataset_dir = dataset_dir + self.input_data_dir = input_data_dir + self.dsd_dir = dsd_dir + + # Standard directories derived from the dataset directory + self.processed_dir = os.path.join(dataset_dir, "processed_data") + self.pvmap_dir = os.path.join(dataset_dir, "pvmap") + self.dc_generated_dir = os.path.join(dataset_dir, "dc_generated") + self.schema_dir = os.path.join(dataset_dir, "schema") + + os.makedirs(log_dir, exist_ok=True) + # Name logs like: rule1_sdg_q1-2026_validation.log + rule_name_slug = self.__class__.__name__.replace('Validator', '').lower() + self.log_file = os.path.join(log_dir, f"{rule_name_slug}_{self.dataset_name}_validation.log") + + def setup_logging(self, rule_title: str): + with open(self.log_file, 'w', encoding='utf-8') as log: + log.write(f"--- {rule_title} Validation Log ---\n") + log.write(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + log.write(f"Dataset: {self.dataset_name}\n") + log.write(f"Dataset Directory: {self.dataset_dir}\n\n") + + def write_log(self, message: str): + with open(self.log_file, 'a', encoding='utf-8') as log: + log.write(f"{message}\n") + + @abstractmethod + def validate(self): + """Implement the core validation logic here.""" + pass diff --git a/scripts/un/un_dataset_validator/scripts/run_isolated_sdg_test.py b/scripts/un/un_dataset_validator/scripts/run_isolated_sdg_test.py new file mode 100644 index 0000000000..17a570db1a --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/run_isolated_sdg_test.py @@ -0,0 +1,88 @@ +import os +import sys +import shutil + +def main(): + base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) + sdg_dir = os.path.join(base_dir, "dcp_data_20260616/20260616/sdg_q1-2026") + test_data_dir = os.path.join(base_dir, "un_dataset_validator/test_data") + + # Target files + csv_file = os.path.join(test_data_dir, "SDG_q1-2026_OBS_AG_FLS_PCT_data.csv") + mcf_file = os.path.join(test_data_dir, "SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf") + + if not os.path.exists(csv_file) or not os.path.exists(mcf_file): + print(f"Error: Could not find test files. Ensure you are running this from the workspace root.") + sys.exit(1) + + # Setup isolated test environment + isolated_dir = os.path.join(base_dir, "un_dataset_validator/isolated_test_env") + processed_dir = os.path.join(isolated_dir, "processed_data") + logs_dir = os.path.join(isolated_dir, "isolated_logs") + + if os.path.exists(isolated_dir): + shutil.rmtree(isolated_dir) + + os.makedirs(processed_dir) + os.makedirs(logs_dir) + + # Copy targeted test files + shutil.copy(csv_file, processed_dir) + shutil.copy(mcf_file, processed_dir) + + # Symlink required dependency directories from the actual dataset + for folder in ["pvmap", "schema", "dc_generated"]: + src = os.path.join(sdg_dir, folder) + dst = os.path.join(isolated_dir, folder) + if os.path.exists(src): + os.symlink(src, dst) + + print(f"✅ Isolated environment created at: {isolated_dir}") + print(f"✅ Isolated logs will be stored in: {logs_dir}\n") + + # Dynamically load the un_dataset_validator rules + sys.path.append(os.path.join(base_dir, "un_dataset_validator/scripts")) + + from test_rule_1 import Rule1Validator + from test_rule_2 import Rule2Validator + from test_rule_3 import Rule3Validator + from test_rule_4 import Rule4Validator + from test_rule_5_7 import Rule5And7Validator + from test_rule_6 import Rule6Validator + from test_rule_9 import Rule9Validator + from test_rule_10 import Rule10Validator + from test_rule_11 import Rule11Validator + from test_rule_12 import Rule12Validator + from test_rule_13 import Rule13Validator + from test_rule_8 import Rule8Validator + from test_rule_14 import Rule14Validator + + validators = [ + Rule1Validator, Rule2Validator, Rule3Validator, Rule4Validator, + Rule5And7Validator, Rule6Validator, Rule8Validator, Rule9Validator, Rule10Validator, Rule11Validator, + Rule12Validator, Rule13Validator, Rule14Validator + ] + + all_passed = True + for v_class in validators: + rule_name = v_class.__name__.replace('Validator', '') + # Instantiate validator pointing to the isolated directory & custom log folder + v = v_class("sdg_q1-2026", isolated_dir, log_dir=logs_dir) + print(f"--- Running {rule_name} ---") + try: + passed = v.validate() + if not passed: + all_passed = False + except Exception as e: + print(f"Error executing {rule_name}: {str(e)}") + all_passed = False + + print("\n==============================") + if all_passed: + print("✅ All un_dataset_validators PASSED on the isolated test files.") + else: + print(f"❌ Some un_dataset_validators FAILED. Please review the specific logs inside:\n {logs_dir}") + print("==============================") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/run_validations.py b/scripts/un/un_dataset_validator/scripts/run_validations.py new file mode 100644 index 0000000000..052c47bc3a --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/run_validations.py @@ -0,0 +1,89 @@ +import argparse +import sys +import os + +# Ensure the scripts directory is in sys.path +sys.path.append(os.path.dirname(__file__)) + +from test_rule_1 import Rule1Validator +from test_rule_2 import Rule2Validator +from test_rule_3 import Rule3Validator +from test_rule_4 import Rule4Validator +from test_rule_5_7 import Rule5And7Validator +from test_rule_6 import Rule6Validator +from test_rule_8 import Rule8Validator +from test_rule_9 import Rule9Validator +from test_rule_10 import Rule10Validator +from test_rule_11 import Rule11Validator +from test_rule_12 import Rule12Validator +from test_rule_13 import Rule13Validator +from test_rule_14 import Rule14Validator +from test_rule_15 import Rule15Validator +from test_rule_16 import Rule16Validator +from test_rule_17 import Rule17Validator +from summary_generator import generate_summary_md + +def main(): + parser = argparse.ArgumentParser(description="UN Data Commons Validation Runner") + parser.add_argument("--dataset", required=True, help="Dataset name, e.g., sdg_q1-2026") + parser.add_argument("--dataset_dir", required=True, help="Dataset directory, e.g., dcp_data_20260616/20260616/sdg_q1-2026") + parser.add_argument("--input_data_dir", required=True, help="Path to raw input DATA directory") + parser.add_argument("--dsd_dir", required=True, help="Path to DSD schemas directory") + parser.add_argument("--rule", required=True, help="Rule to run (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, or all)") + parser.add_argument("--log_dir", default="un_dataset_validator/logs", help="Directory to save logs") + args = parser.parse_args() + + validators = { + "1": Rule1Validator, + "2": Rule2Validator, + "3": Rule3Validator, + "4": Rule4Validator, + "5": Rule5And7Validator, + "7": Rule5And7Validator, # Support calling rule 7 individually as alias + "6": Rule6Validator, + "8": Rule8Validator, + "9": Rule9Validator, + "10": Rule10Validator, + "11": Rule11Validator, + "12": Rule12Validator, + "13": Rule13Validator, + "14": Rule14Validator, + "15": Rule15Validator, + "16": Rule16Validator, + "17": Rule17Validator, + } + + if args.rule.lower() == "all": + # Run rules 1–6, 8–17 in order (6.1, 10.1, 13.1 sub-rules are temporarily excluded) + rules_to_run = ["1", "2", "3", "4", "5", "6", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17"] + else: + rules_to_run = [r.strip() for r in args.rule.split(",")] + + all_passed = True + for rule in rules_to_run: + if rule not in validators: + print(f"Unknown rule: {rule}") + sys.exit(1) + + validator_class = validators[rule] + validator = validator_class(args.dataset, args.dataset_dir, input_data_dir=args.input_data_dir, dsd_dir=args.dsd_dir, log_dir=args.log_dir) + print(f"--- Running Rule {rule} ---") + passed = validator.validate() + if not passed: + all_passed = False + + if args.rule.lower() == "all": + try: + generate_summary_md(args.dataset, args.log_dir) + except Exception as e: + print(f"Error generating summary.md: {e}") + + if not all_passed: + print("\nSome validations FAILED. Check logs for details.") + sys.exit(1) + else: + print("\nAll validations PASSED.") + +if __name__ == "__main__": + main() + diff --git a/scripts/un/un_dataset_validator/scripts/summary_generator.py b/scripts/un/un_dataset_validator/scripts/summary_generator.py new file mode 100644 index 0000000000..2b334c5db5 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/summary_generator.py @@ -0,0 +1,149 @@ +import os +import re +from datetime import datetime + +def extract_rule_num(filename): + # Matches rule1, rule5and7, rule10, etc. + m = re.match(r'rule(\d+)(?:and(\d+))?', filename) + if m: + return (int(m.group(1)), filename) + return (999, filename) + +def generate_summary_md(dataset_name: str, log_dir: str): + output_path = os.path.join(log_dir, "summary.md") + + # Locate all logs for this dataset in log_dir + files = [] + if os.path.exists(log_dir): + for f in os.listdir(log_dir): + if f.endswith(".log") and f.startswith("rule") and f"_{dataset_name}_" in f: + files.append(f) + + files.sort(key=extract_rule_num) + + if not files: + print(f"No validation log files found for dataset '{dataset_name}' in directory '{log_dir}'. Skipping summary.md generation.") + return + + summary_lines = [] + summary_lines.append(f"# Validation Summary: {dataset_name}") + summary_lines.append("") + summary_lines.append(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ") + summary_lines.append(f"**Log Directory:** `{log_dir}` ") + summary_lines.append("") + summary_lines.append("---") + summary_lines.append("") + + stats = { + "PASSED": 0, + "FAILED": 0, + "TOTAL": 0 + } + + matrix_lines = [] + matrix_lines.append("| Rule | Status | Summary |") + matrix_lines.append("| :--- | :---: | :--- |") + + details_block = [] + + for filename in files: + filepath = os.path.join(log_dir, filename) + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + + # Determine Rule Title + rule_header_match = re.search(r'--- (Rule [^ ]+(?: & \d+)?[^\-]+) ---', content) + if rule_header_match: + rule_name = rule_header_match.group(1).strip() + else: + rule_name = filename.replace(f"_{dataset_name}_validation.log", "").replace("_", " ").title() + + # Determine overall Status + status = "PASSED" + if "FAILED" in content or "FAILED" in filename or ("errors" in content.lower() and "No violations detected" not in content and "No global 1:1 mapping violations" not in content and "No validation failures detected" not in content): + if "Overall Status: PASSED" in content: + status = "PASSED" + else: + status = "FAILED" + + if "Overall Status: FAILED" in content: + status = "FAILED" + elif "Overall Status: PASSED" in content: + status = "PASSED" + + stats[status] += 1 + stats["TOTAL"] += 1 + + # Extract Summary section + validation_summary_section = "" + summary_idx = content.find("--- Validation Summary ---") + if summary_idx == -1: + summary_idx = content.find("Summary:") + + if summary_idx != -1: + validation_summary_section = content[summary_idx:].strip() + # clean up lines + val_lines = [l.strip() for l in validation_summary_section.split("\n") if l.strip() and not l.startswith("---") and not l.startswith("Summary:")] + validation_summary_section = " | ".join(val_lines) + else: + # Check for Overall Status or fallback + lines = content.split("\n") + status_lines = [l.strip() for l in lines if "overall status" in l.lower() or "validation results" in l.lower()] + if status_lines: + validation_summary_section = " | ".join(status_lines) + else: + validation_summary_section = "Completed validation execution." + + # Append to Matrix table + status_badge = f"**PASSED**" if status == "PASSED" else f"**FAILED**" + matrix_lines.append(f"| {rule_name} | {status_badge} | {validation_summary_section} |") + + # Build Detailed Breakdown + details_block.append(f"### {rule_name}") + details_block.append("") + details_block.append(f"- **Status:** {status_badge}") + if validation_summary_section: + details_block.append(f"- **Summary:** {validation_summary_section}") + + # Parse failure report details if FAILED + if status == "FAILED": + details_block.append("- **Failure Details:**") + details_block.append(" ```text") + failure_section_match = re.search(r'--- Detailed Failure Report ---\s*(.*?)(?=\n---|\Z)', content, re.DOTALL) + if failure_section_match: + fail_details = failure_section_match.group(1).strip().split("\n") + # Show up to 15 lines of errors + for fd in fail_details[:15]: + details_block.append(f" {fd}") + if len(fail_details) > 15: + details_block.append(f" ... and {len(fail_details) - 15} more lines of errors.") + else: + # Fallback: grab some error messages + lines = content.split("\n") + error_lines = [l for l in lines if "failed" in l.lower() or "error" in l.lower() or "invalid" in l.lower() or "missing" in l.lower()] + for el in error_lines[:10]: + details_block.append(f" {el}") + if len(error_lines) > 10: + details_block.append(f" ... and {len(error_lines) - 10} more errors.") + details_block.append(" ```") + + details_block.append("") + details_block.append("---") + details_block.append("") + + summary_lines.append("## Overall Status Matrix") + summary_lines.append("") + summary_lines.extend(matrix_lines) + summary_lines.append("") + summary_lines.append(f"**Total Rules Run:** {stats['TOTAL']} | **Passed:** {stats['PASSED']} | **Failed:** {stats['FAILED']} ") + summary_lines.append("") + summary_lines.append("---") + summary_lines.append("") + summary_lines.append("## Detailed Rule Breakdowns") + summary_lines.append("") + summary_lines.extend(details_block) + + with open(output_path, "w", encoding="utf-8") as f: + f.write("\n".join(summary_lines)) + + print(f"Successfully generated validation summary report at: {output_path}") diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_1.py b/scripts/un/un_dataset_validator/scripts/test_rule_1.py new file mode 100644 index 0000000000..c36a63cd91 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_1.py @@ -0,0 +1,128 @@ +""" +VALIDATION RULE: +================ +# Rule 1: 1:1 Mapping Validation (PV Maps) + +## Requirement +Ensure all UN concepts and codes have a strict 1:1 mapping with the agency-specific DCP schema. + +## Context & Rules (From Meeting Notes) +- There must be no duplicate properties assigned to a single concept. +- Within the Property-Value (PV) maps, the `event code` column must align perfectly with the `constraint property`. +- For example, if the concept is "age" or "poverty status", the `event code` must be "age" or "poverty status" exactly, maintaining consistency across the file. +- This rule applies strictly to the agency-specific schema rather than the base Data Commons mapping. + + +""" +import os +import sys +import glob +import csv + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import to_canonical_format + +class Rule1Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 1 (1:1 Schema Mapping)") + + pattern = os.path.join(self.pvmap_dir, "CL_*_pvmap.csv") + pvmap_files = glob.glob(pattern) + + if not pvmap_files: + self.write_log(f"No CL_*_pvmap.csv files found in {self.pvmap_dir}") + print(f"No CL_*_pvmap.csv files found in {self.pvmap_dir}. Log written to {self.log_file}") + return False + + total_files = len(pvmap_files) + failed_files = 0 + + concept_to_prop = {} + prop_to_concept = {} + + global_errors = [] + file_specific_errors = {} + + for filepath in pvmap_files: + filename = os.path.basename(filepath) + file_errors = [] + + with open(filepath, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + if 'UnConcept' not in reader.fieldnames or 'ConstraintProp' not in reader.fieldnames: + file_errors.append("Missing required columns: 'UnConcept' or 'ConstraintProp'") + file_specific_errors[filename] = file_errors + failed_files += 1 + continue + + for row_num, row in enumerate(reader, start=2): + un_concept = row.get('UnConcept', '').strip() + constraint_prop = row.get('ConstraintProp', '').strip() + + if not un_concept or not constraint_prop: + continue + + canon_concept = to_canonical_format(un_concept) + canon_prop = to_canonical_format(constraint_prop) + + if canon_concept in ['series', 'geography', 'timeperiod', 'obsvalue']: + continue + + if canon_concept != canon_prop: + file_errors.append(f"File {filename}, Row {row_num}: Alignment mismatch. Canonical UnConcept '{canon_concept}' != Canonical ConstraintProp '{canon_prop}'. Original: '{un_concept}' vs '{constraint_prop}'") + + if un_concept in concept_to_prop: + if concept_to_prop[un_concept] != constraint_prop: + global_errors.append(f"Duplicate assignment for UnConcept '{un_concept}': mapped to both '{concept_to_prop[un_concept]}' and '{constraint_prop}' (found in {filename})") + else: + concept_to_prop[un_concept] = constraint_prop + + if constraint_prop in prop_to_concept: + if prop_to_concept[constraint_prop] != un_concept: + global_errors.append(f"Duplicate assignment for ConstraintProp '{constraint_prop}': mapped to both '{prop_to_concept[constraint_prop]}' and '{un_concept}' (found in {filename})") + else: + prop_to_concept[constraint_prop] = un_concept + + if file_errors: + file_specific_errors[filename] = file_errors + failed_files += 1 + + self.write_log(f"--- Global 1:1 Mapping Violations ---") + if not global_errors: + self.write_log("No global 1:1 mapping violations detected.") + else: + for err in global_errors: + self.write_log(f"- {err}") + + self.write_log(f"\n--- File-Specific Alignment Violations ---") + if not file_specific_errors: + self.write_log("No alignment violations detected.") + else: + for filename, errors in file_specific_errors.items(): + self.write_log(f"\nFAILED FILE: {filename}") + for err in errors[:20]: + self.write_log(f" - {err}") + if len(errors) > 20: + self.write_log(f" ... and {len(errors) - 20} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total_files}") + self.write_log(f"Files with alignment errors: {len(file_specific_errors)}") + self.write_log(f"Total global 1:1 violations: {len(global_errors)}") + + status = "PASSED" if not global_errors and not file_specific_errors else "FAILED" + self.write_log(f"Overall Status: {status}") + print(f"Rule 1 Validation complete. Results written to {self.log_file}") + + return status == "PASSED" + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_1.py ") + sys.exit(1) + validator = Rule1Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_10.py b/scripts/un/un_dataset_validator/scripts/test_rule_10.py new file mode 100644 index 0000000000..2dc159a6ea --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_10.py @@ -0,0 +1,168 @@ +""" +VALIDATION RULE: +================ +# Rule 10: Attributes as Output Columns + +## Description +According to the UN Data Commons mapping rules, every concept defined in the dataset's Data Structure Definition (DSD) file with a `ROLE` of `Attribute` must be present as a separate column in the output `data.csv` file. + +Attributes provide supplementary information (such as footnotes, observation statuses, or flags) about the observation value. Unlike dimensions, attributes must **NOT** be attached as properties to the Statistical Variables. + +**Note on Exclusions (Rule 10.1):** Parsing and validating the internal string structures of complex attributes (e.g., verifying comma-separated multiple footnotes inside a single `FOOTNOTE` cell) is marked as an "Ask Ajai" item and is strictly excluded from this validation check. This rule only validates the *presence* of the attribute column in the output. + +## Files Involved +- **Input:** Dataset-specific DSD file (e.g., `schema/dsd.csv` or similar file defining `ROLE`). +- **Output:** The generated data CSV file (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv`). + + +# Rule 10.1: Coded Attributes Mapping to Property Enum + +## Requirement +All coded attributes within the dataset must be correctly mapped and assigned a `property:enum` value in the Data Commons Project (DCP) schema. + +## Context & Rules +- According to Rule 10, all attributes (columns marked as 'Attribute' in the DSD) should become output columns along with the observation. +- For attributes that are specifically *coded* (meaning they pull from a defined codelist or restricted set of values, as opposed to free-text), the schema must reflect this by assigning a `property:enum` mapping. +- This ensures that coded attributes maintain their structural integrity and defined value set in the output DCP schema. + + +""" +import os +import sys +import glob +import csv +import re + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator + +def sanitize_for_match(text: str) -> str: + """Removes underscores and converts to lowercase for column matching.""" + return text.replace("_", "").lower() + +class Rule10Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 10 (Attributes as Output Columns)") + + if not self.dsd_dir or not os.path.exists(self.dsd_dir): + self.write_log(f"Could not find DSD directory for dataset {self.dataset_name}") + print(f"Could not find DSD directory for dataset {self.dataset_name}") + return False + + dsd_dir = self.dsd_dir + dsd_files = glob.glob(os.path.join(dsd_dir, "*.csv")) + + if not dsd_files: + self.write_log(f"No DSD files found in {dsd_dir}") + return False + + total_files = len(dsd_files) + failed_files = 0 + + self.write_log(f"DSD Directory: {dsd_dir}") + self.write_log(f"Output Directory: {self.processed_dir}\n") + + # Rule 10.1: Load schema to check Enum mapping + schema_file = os.path.join(self.dataset_dir, "schema", f"un_codelist_schema_{self.dataset_name}.mcf") + schema_content = "" + if os.path.exists(schema_file): + with open(schema_file, 'r', encoding='utf-8') as f: + schema_content = f.read() + + for dsd_path in sorted(dsd_files): + filename = os.path.basename(dsd_path) + match = re.search(r'_DSD_(.*?)\.csv', filename) + if not match: + continue + series = match.group(1) + + csv_pattern = os.path.join(self.processed_dir, f"*_OBS_{series}_data.csv") + csv_files = glob.glob(csv_pattern) + + if not csv_files: + self.write_log(f"[{series}] SKIPPED: Missing corresponding CSV file.") + continue + + csv_path = csv_files[0] + errors = [] + attributes = [] + coded_attributes = [] + + with open(dsd_path, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + for row in reader: + role = row.get('ROLE', '').strip().lower() + concept = row.get('CONCEPT', '').strip() + repr_type = row.get('REPRESENTATION', '').strip().lower() + + if role == 'attribute': + attributes.append(concept) + if repr_type == 'coded': + coded_attributes.append(concept) + + with open(csv_path, 'r', encoding='utf-8') as f: + csv_reader = csv.reader(f) + headers = next(csv_reader, []) + sanitized_headers = [sanitize_for_match(h) for h in headers] + + attr_column_map = { + 'UNIT_MEASURE': 'unit', + 'FREQUENCY': 'opservationperiod', # Pipeline typo + 'UNIT_MULT': None, # Consumed by Rule 7, not in CSV + } + + for attr in attributes: + attr_upper = attr.upper() + + # Rule 10.1: Check Enum mapping for coded attributes + if attr in coded_attributes: + if schema_content: + # Find the node for this unConcept + concept_pattern = rf"Node: dcid:([^\n]+)\n(?:[^\n]+\n)*?unConcept: \"{attr_upper}\"\n" + # Wait, the node might have unConcept before or after rangeIncludes + # Let's search the whole block + block_pattern = rf"Node: dcid:[^\n]+(?:\n(?!\n).*)*unConcept: \"{attr_upper}\"(?:\n(?!\n).*)*" + match_block = re.search(block_pattern, schema_content) + if match_block: + block_text = match_block.group(0) + if "rangeIncludes: dcid:" not in block_text: + errors.append(f"File {os.path.basename(csv_path)}: Rule 10.1 Violation - Coded attribute '{attr}' does not have a 'rangeIncludes' enum mapping in the schema.") + else: + # It might be defined differently (e.g. unit) + if attr_upper not in ['UNIT_MEASURE', 'UNIT_MULT', 'FREQUENCY']: + errors.append(f"File {os.path.basename(csv_path)}: Rule 10.1 Violation - Coded attribute '{attr}' schema definition not found in un_codelist_schema.") + + if attr_upper == 'UNIT_MULT' or (attr_column_map.get(attr_upper) is None and attr_upper in attr_column_map): + continue + + expected_col = attr_column_map.get(attr_upper, sanitize_for_match(attr)) + if expected_col not in sanitized_headers: + errors.append(f"File {os.path.basename(csv_path)}: Attribute '{attr}' not found as a separate column in the output CSV. Expected a column matching '{expected_col}'.") + + if errors: + failed_files += 1 + self.write_log(f"[{series}] FAILED") + for err in errors: + self.write_log(f" - {err}") + self.write_log("") + else: + self.write_log(f"[{series}] PASSED") + + self.write_log(f"\nSummary:") + self.write_log(f"Total DSDs Checked: {total_files}") + self.write_log(f"Passed: {total_files - failed_files}") + self.write_log(f"Failed: {failed_files}") + + print(f"Rule 10 Validation completed. {failed_files}/{total_files} files failed.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_10.py ") + sys.exit(1) + validator = Rule10Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_11.py b/scripts/un/un_dataset_validator/scripts/test_rule_11.py new file mode 100644 index 0000000000..af58ce4d3f --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_11.py @@ -0,0 +1,94 @@ +""" +VALIDATION RULE: +================ +# Rule 11: StatVar DCID Format & Special Characters + +## Overview +This rule validates the structural format of the Data Commons Identifier (DCID) generated for each Statistical Variable (StatVar). The DCID must strictly adhere to a specific templated format, and any illegal or special characters within the originating codes must be converted to underscores (`_`) to ensure valid identifier syntax. + +## Files Involved +- **Output:** `output_stat_vars.mcf` (Specifically examining the `Node: dcid:...` lines). + + +""" +import os +import sys +import glob +import re + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) +from base_validator import BaseRuleValidator + +class Rule11Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 11 (StatVar DCID Format)") + + pattern = os.path.join(self.processed_dir, "*_stat_vars.mcf") + mcf_files = glob.glob(pattern) + + if not mcf_files: + self.write_log(f"No *_stat_vars.mcf files found in {self.processed_dir}") + print(f"No *_stat_vars.mcf files found in {self.processed_dir}. Log written to {self.log_file}") + return False + + total = len(mcf_files) + passed = 0 + failed = 0 + failed_details = [] + + # Regex to match the expected format: + # dcid:undata//[.--__...] + # We allow a-zA-Z, 0-9, and underscores. + dcid_regex = re.compile(r"^dcid:[a-zA-Z0-9_]+/[a-zA-Z0-9_]+/[A-Z0-9_]+(\.[A-Z0-9_]+--[a-zA-Z0-9_]+(__[A-Z0-9_]+--[a-zA-Z0-9_]+)*)?$") + + for filepath in mcf_files: + filename = os.path.basename(filepath) + errors = [] + + with open(filepath, 'r', encoding='utf-8') as f: + for line_idx, line in enumerate(f, start=1): + line = line.strip() + if line.startswith("Node: dcid:"): + dcid = line[6:] # Strip 'Node: ' + + if not dcid_regex.match(dcid): + errors.append(f"File {filename}, Line {line_idx}: Invalid DCID format: '{dcid}'") + + if errors: + failed += 1 + failed_details.append({"filename": filename, "errors": errors}) + else: + passed += 1 + + self.write_log(f"--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No validation failures detected.") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total format errors in this file: {len(errors)}") + + limit = min(10, len(errors)) + for err in errors[:limit]: + self.write_log(f" - {err}") + if len(errors) > limit: + self.write_log(f" ... and {len(errors) - limit} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total MCF files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + + status_passed = failed == 0 + self.write_log(f"Overall Status: {'PASSED' if status_passed else 'FAILED'}") + print(f"Rule 11 Validation complete. Results written to {self.log_file}") + return status_passed + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_11.py ") + sys.exit(1) + validator = Rule11Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_12.py b/scripts/un/un_dataset_validator/scripts/test_rule_12.py new file mode 100644 index 0000000000..615dfdd5f3 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_12.py @@ -0,0 +1,149 @@ +""" +VALIDATION RULE: +================ +# Rule 12: Names (alternateName, Value names, nameWithLanguage) + +## 1. Rule Description +* **Core Rule (12):** This rule ensures that names for properties and values generated in the schema accurately reflect their source definitions in the DSD (Data Structure Definition) and Codelists. +* **Sub-rule (12.1):** A property's `alternateName` must match the corresponding name defined in the DSD file. +* **Sub-rule (12.2):** A value's name must match the name defined in the specific concept's codelist (`CL` file). +* **Sub-rule (12.3):** Any names available in languages other than the default must be appropriately added to the `nameWithLanguage` property. + +## 2. Files Involved +* **Data Input:** The transcoded dataset. +* **DSD File:** Defines the structural metadata and concepts. +* **Codelists (CL):** Define the valid values and their corresponding names for each concept. +* **Output MCF Files:** The generated schema and stat vars where these properties are defined. + + +""" +import os +import sys +import glob +import csv +import string + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) +from base_validator import BaseRuleValidator + +def normalize_name(s): + if not s: + return "" + # Remove all punctuation and lowercase + s = s.translate(str.maketrans('', '', string.punctuation)).lower() + # Normalize spaces + return " ".join(s.split()) + +class Rule12Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 12 (Names Validation)") + + # 1. Load all PV Maps to get expected names + expected_names = {} + pvmap_files = glob.glob(os.path.join(self.pvmap_dir, "CL_*_pvmap.csv")) + for pvmap_file in pvmap_files: + with open(pvmap_file, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + dcid = row.get('ConstraintPropValue', '').strip() + name = row.get('ConstraintValueName', '').strip().strip('"') + if dcid and name: + expected_names[dcid] = name + + self.write_log(f"Loaded {len(expected_names)} expected value names from PV maps.") + + # 2. Check Schema MCFs for Value names (12.2) + schema_pattern = os.path.join(self.dataset_dir, "schema", "*.mcf") + schema_files = glob.glob(schema_pattern) + + failed = False + errors = [] + warnings = [] + + if not schema_files: + self.write_log("No schema MCF files found.") + failed = True + else: + for schema_file in schema_files: + current_node = None + current_name = None + + with open(schema_file, 'r', encoding='utf-8') as f: + for line_idx, line in enumerate(f, start=1): + line = line.strip() + if line.startswith("Node: "): + # Process previous node + if current_node and current_node in expected_names: + expected = expected_names[current_node] + if current_name is None: + errors.append(f"{os.path.basename(schema_file)}: Missing name for {current_node}. Expected '{expected}'") + elif normalize_name(current_name) != normalize_name(expected): + errors.append(f"{os.path.basename(schema_file)}: Value mismatch for {current_node}. Expected '{expected}', found '{current_name}'") + + current_node = line[6:].strip() + if current_node.startswith('dcid:'): + current_node = current_node[5:] + current_name = None + elif line.startswith("name: "): + # Handle names that might have extra characters or spaces around quotes + extracted_name = line[6:].strip() + if extracted_name.startswith('"') and extracted_name.endswith('"'): + current_name = extracted_name[1:-1] + else: + current_name = extracted_name.strip('"') + + # Check the last node + if current_node and current_node in expected_names: + expected = expected_names[current_node] + if current_name is None: + errors.append(f"{os.path.basename(schema_file)}: Missing name for {current_node}. Expected '{expected}'") + elif normalize_name(current_name) != normalize_name(expected): + errors.append(f"{os.path.basename(schema_file)}: Value mismatch for {current_node}. Expected '{expected}', found '{current_name}'") + + # 3. Check StatVars MCF for missing names + statvar_pattern = os.path.join(self.processed_dir, "*_stat_vars.mcf") + statvar_files = glob.glob(statvar_pattern) + + missing_statvar_names_count = 0 + if statvar_files: + for sv_file in statvar_files: + with open(sv_file, 'r', encoding='utf-8') as f: + content = f.read() + nodes = content.split("Node: ") + for node in nodes[1:]: # Skip the first empty split before first Node + if "typeOf: dcid:StatisticalVariable" in node: + if "\nname: " not in node: + missing_statvar_names_count += 1 + + if missing_statvar_names_count > 0: + warnings.append(f"Found {missing_statvar_names_count} Statistical Variables missing a 'name' property. This is a known pipeline issue (Check 12 / 13).") + + self.write_log("--- Validation Results ---") + if errors: + self.write_log(f"Found {len(errors)} name mismatches:") + for err in errors[:50]: + self.write_log(f" - {err}") + if len(errors) > 50: + self.write_log(f" ... and {len(errors) - 50} more errors.") + failed = True + else: + self.write_log("No value name mismatches found against PV maps (Rule 12.2 passed).") + + if warnings: + self.write_log("\n--- Warnings ---") + for warn in warnings: + self.write_log(f" - {warn}") + + status = "FAILED" if failed else "PASSED" + self.write_log(f"\nOverall Status: {status}") + print(f"Rule 12 Validation complete. Results written to {self.log_file}") + + return not failed + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_12.py ") + sys.exit(1) + validator = Rule12Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_13.py b/scripts/un/un_dataset_validator/scripts/test_rule_13.py new file mode 100644 index 0000000000..c84a42c561 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_13.py @@ -0,0 +1,116 @@ +""" +VALIDATION RULE: +================ +# Rule 13.1: Statvar Name Template + +## Requirement +The name assigned to a Statistical Variable (StatVar) must adhere to a specific template format to ensure consistency and readability across the Data Commons Project (DCP). + +## Context & Rules +- According to the checklist, the required template for a StatVar name is: + `" [=, ...]"` +- This template provides a clear, human-readable summary of the underlying data series and the specific constraint properties (concepts and codes) that define the statistical variable. +- For example, if the series is "Unemployment Rate" and the concepts are "Age" and "Gender" with codes "15-24" and "Female" respectively, the name should be constructed as: + `"Unemployment Rate [Age=15-24, Gender=Female]"` + + +""" +import os +import sys +import glob +import re + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) +from base_validator import BaseRuleValidator + +class Rule13Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 13 (StatVar Name Template)") + + mcf_pattern = os.path.join(self.dataset_dir, "schema", f"*_stat_vars.mcf") + mcf_files = glob.glob(mcf_pattern) + + if not mcf_files: + self.write_log(f"No _stat_vars.mcf files found in {os.path.join(self.dataset_dir, 'schema')}") + print(f"No _stat_vars.mcf files found.") + return False + + total_files = len(mcf_files) + failed_files = 0 + + self.write_log(f"Output Schema Directory: {os.path.join(self.dataset_dir, 'schema')}\n") + + for mcf_path in sorted(mcf_files): + filename = os.path.basename(mcf_path) + errors = [] + + with open(mcf_path, 'r', encoding='utf-8') as f: + content = f.read() + + nodes = content.split('Node: dcid:') + for node in nodes: + if not node.strip(): + continue + if 'typeOf: dcid:StatisticalVariable' not in node and 'typeOf: dcs:StatisticalVariable' not in node: + continue + + name_match = re.search(r'name:\s*"([^"]+)"', node) + if not name_match: + # Depending on pipeline, some generic statvars might not have generated names, but we should flag it + # But wait, some might just be properties. We already filtered by typeOf: StatisticalVariable + errors.append(f"StatVar in {filename} is missing a 'name' property.") + continue + + name_val = name_match.group(1) + + # Check template format: Base Name [Concept=Code, Concept=Code] + # If there are no constraints, it might just be "Base Name" + match = re.match(r'^(.+?)(?:\s+\[(.+)\])?$', name_val) + if not match: + errors.append(f"StatVar name '{name_val}' in {filename} does not match expected overall format.") + continue + + constraints_str = match.group(2) + if constraints_str: + # Look for Concept=Code patterns, handling commas in Code + # Assumes Concept does not contain '=' or ',' + # and that concepts are separated by ', ' followed by a new Concept= + constraint_matches = re.finditer(r'([^,=]+)=(.+?)(?=(?:, [^,=]+=)|$)', constraints_str) + + found_constraints = False + for c_match in constraint_matches: + found_constraints = True + concept = c_match.group(1).strip() + code = c_match.group(2).strip() + if not concept or not code: + errors.append(f"Constraint in '{name_val}' has empty concept or code.") + + if not found_constraints: + errors.append(f"StatVar name '{name_val}' has brackets but no 'Concept=Code' format inside.") + + if errors: + failed_files += 1 + self.write_log(f"[{filename}] FAILED") + for err in errors: + self.write_log(f" - {err}") + self.write_log("") + else: + self.write_log(f"[{filename}] PASSED") + + self.write_log(f"\nSummary:") + self.write_log(f"Total MCFs Checked: {total_files}") + self.write_log(f"Passed: {total_files - failed_files}") + self.write_log(f"Failed: {failed_files}") + + print(f"Rule 13 Validation completed. {failed_files}/{total_files} files failed.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_13.py ") + sys.exit(1) + validator = Rule13Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_14.py b/scripts/un/un_dataset_validator/scripts/test_rule_14.py new file mode 100644 index 0000000000..fb70dbdbaa --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_14.py @@ -0,0 +1,111 @@ +""" +VALIDATION RULE: +================ +# Rule 14: Duplicate Checks +Ensures there are no duplicated rows across the dataset observation files. +""" +import os +import glob +import pandas as pd +from base_validator import BaseRuleValidator + +class Rule14Validator(BaseRuleValidator): + """ + Rule 14 Validator: Checks for duplicate observations in the generated CSV files. + + A duplicate observation is defined as multiple rows having the same combination of: + - variableMeasured (StatVar) + - observationAbout (Place) + - observationDate (Date) + """ + + def validate(self): + self.setup_logging("Rule 14 (Observation Uniqueness)") + + pattern = os.path.join(self.processed_dir, "*_data.csv") + csv_files = glob.glob(pattern) + + if not csv_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}. Log written to {self.log_file}") + return False + + total_files = len(csv_files) + failed_files = 0 + total_duplicates = 0 + + # Columns that make up the unique composite key in Data Commons + key_cols = ['variableMeasured', 'observationAbout', 'observationDate'] + + for filepath in csv_files: + filename = os.path.basename(filepath) + + try: + # Read CSV, forcing all to string to prevent type issues + df = pd.read_csv(filepath, dtype=str) + + # Check if the required columns exist + missing_cols = [col for col in key_cols if col not in df.columns] + if missing_cols: + self.write_log(f"File {filename}: Missing required columns for uniqueness check: {missing_cols}") + failed_files += 1 + continue + + # Find duplicates + dups = df[df.duplicated(subset=key_cols, keep=False)] + + if not dups.empty: + failed_files += 1 + dup_count = len(dups) + total_duplicates += dup_count + + self.write_log(f"\nFAILED FILE: {filename}") + self.write_log(f"Found {dup_count} duplicate rows.") + + # Sort so duplicates are grouped together in the log + dups_sorted = dups.sort_values(by=key_cols) + + # Log up to the first 20 duplicate lines for context + # The index + 2 assumes a 1-based header (index 0 is data row 1 -> line 2) + count = 0 + for idx, row in dups_sorted.iterrows(): + if count >= 20: + self.write_log(f" ... and {dup_count - 20} more duplicate rows.") + break + + line_num = idx + 2 + stat_var = row['variableMeasured'] + place = row['observationAbout'] + date = row['observationDate'] + val = row.get('value', 'N/A') + + self.write_log(f" - File {filename}, Line {line_num}: Duplicate observation -> StatVar: '{stat_var}', Place: '{place}', Date: '{date}', Value: '{val}'") + count += 1 + + except Exception as e: + self.write_log(f"File {filename}: Error processing file: {e}") + failed_files += 1 + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total output files processed: {total_files}") + self.write_log(f"Files with duplicates: {failed_files}") + self.write_log(f"Total duplicate rows detected: {total_duplicates}") + + status = "PASSED" if failed_files == 0 else "FAILED" + self.write_log(f"Overall Status: {status}") + + print(f"Rule 14 Validation completed. {failed_files}/{total_files} files failed with duplicates.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + import sys + import argparse + parser = argparse.ArgumentParser(description="Run Rule 14 Validation (Observation Uniqueness)") + parser.add_argument("--dataset", required=True, help="Dataset name (e.g., sdg_q1-2026)") + parser.add_argument("--dataset_dir", required=True, help="Path to the dataset directory") + args = parser.parse_args() + + validator = Rule14Validator(args.dataset, args.dataset_dir) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_15.py b/scripts/un/un_dataset_validator/scripts/test_rule_15.py new file mode 100644 index 0000000000..f10be5d074 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_15.py @@ -0,0 +1,116 @@ +""" +VALIDATION RULE: +================ +# Rule 15: Duplicate Properties in Statistical Variables + +## Requirement +Ensure there are no repeated property or concept codes assigned within a single Statistical Variable definition in the generated MCF files. + +## Context & Rules +- Read `*_stat_vars.mcf` files in the output directory. +- For each `Node: dcid:...` block, track all the properties defined. +- If any property key (e.g., `age:`, `measurementMethod:`) appears more than once for a single node, flag it as an error. +""" + +import os +import glob +import sys + +# Ensure the scripts directory is in sys.path +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator + +class Rule15Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 15 (Duplicate StatVar Properties)") + + # Look for MCF files in the processed directory + pattern = os.path.join(self.processed_dir, "*_stat_vars.mcf") + mcf_files = glob.glob(pattern) + + # Also check the dataset root if they are stored there + if not mcf_files: + pattern = os.path.join(self.dataset_dir, "*_stat_vars.mcf") + mcf_files = glob.glob(pattern) + + if not mcf_files: + # Fallback to dc_generated if present + pattern = os.path.join(self.dc_generated_dir, "*_stat_vars.mcf") + mcf_files = glob.glob(pattern) + + if not mcf_files: + self.write_log(f"No *_stat_vars.mcf files found in {self.processed_dir}, {self.dataset_dir}, or {self.dc_generated_dir}") + print(f"No *_stat_vars.mcf files found. Log written to {self.log_file}") + return False + + total_files = len(mcf_files) + failed_files = 0 + total_violations = 0 + + for filepath in mcf_files: + filename = os.path.basename(filepath) + file_errors = [] + + try: + with open(filepath, 'r', encoding='utf-8') as f: + current_node = None + seen_properties = set() + + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line or line.startswith('#'): + continue + + if line.startswith('Node:'): + current_node = line.split('Node:')[1].strip() + seen_properties = set() + continue + + if current_node and ':' in line: + # Split on the first colon to get the property key + parts = line.split(':', 1) + prop_key = parts[0].strip() + + # Only track properties, not comments or multiline continuations without colons + if prop_key in seen_properties: + error_msg = f"File {filename}, Line {line_num}: Duplicate property '{prop_key}' found in Node '{current_node}'" + file_errors.append(error_msg) + total_violations += 1 + else: + seen_properties.add(prop_key) + + except Exception as e: + file_errors.append(f"File {filename}: Error processing file: {e}") + + if file_errors: + failed_files += 1 + self.write_log(f"\nFAILED FILE: {filename}") + # Log up to 20 errors per file + for err in file_errors[:20]: + self.write_log(f" - {err}") + if len(file_errors) > 20: + self.write_log(f" ... and {len(file_errors) - 20} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total output files processed: {total_files}") + self.write_log(f"Files with duplicate properties: {failed_files}") + self.write_log(f"Total duplicate properties detected: {total_violations}") + + status = "PASSED" if failed_files == 0 else "FAILED" + self.write_log(f"Overall Status: {status}") + + print(f"Rule 15 Validation completed. {failed_files}/{total_files} files failed.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="Run Rule 15 Validation (Duplicate StatVar Properties)") + parser.add_argument("--dataset", required=True, help="Dataset name (e.g., sdg_q1-2026)") + parser.add_argument("--dataset_dir", required=True, help="Path to the dataset directory") + args = parser.parse_args() + + validator = Rule15Validator(args.dataset, args.dataset_dir) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_16.py b/scripts/un/un_dataset_validator/scripts/test_rule_16.py new file mode 100644 index 0000000000..35e9830496 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_16.py @@ -0,0 +1,147 @@ +""" +VALIDATION RULE: +================ +# Rule 16: Observation Completeness (#input tracking) + +## Requirement +Verify that no non-empty observations are improperly dropped during processing. The generated `#input` column tracks the origin of each data point, so every valid observation cell in the source data should have a corresponding `#input` reference in the output. + +## Context & Rules +- For every `*_data.csv` in the output directory, collect all the `#input` coordinates. +- For every `.csv` in the input data directory, parse the data. +- Identify cells that should have been processed (non-empty data points). +- Ensure that the generated output tracks these cells. +""" + +import os +import glob +import sys +import pandas as pd + +# Ensure the scripts directory is in sys.path +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator + +class Rule16Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 16 (Observation Completeness)") + + if not self.input_data_dir: + self.write_log("Error: input_data_dir is required for Rule 16.") + print(f"Error: input_data_dir is required for Rule 16. Log written to {self.log_file}") + return False + + # 1. Collect all `#input` references from output files + out_pattern = os.path.join(self.processed_dir, "*_data.csv") + out_csv_files = glob.glob(out_pattern) + + if not out_csv_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + return False + + actual_inputs = set() + for out_file in out_csv_files: + try: + # Read output CSV safely + df = pd.read_csv(out_file, dtype=str) + if '#input' in df.columns: + # Dropna and strip to be safe + inputs = df['#input'].dropna().str.strip() + for item in inputs: + if item: + actual_inputs.add(item) + except Exception as e: + self.write_log(f"Error reading output file {out_file}: {e}") + + self.write_log(f"Extracted {len(actual_inputs)} unique #input references from output data.") + + # 2. Iterate through input files and check if non-empty observations exist in actual_inputs + in_pattern = os.path.join(self.input_data_dir, "*.csv") + in_csv_files = glob.glob(in_pattern) + + if not in_csv_files: + self.write_log(f"No input .csv files found in {self.input_data_dir}") + return False + + failed_files = 0 + total_missing = 0 + + for in_file in in_csv_files: + filename = os.path.basename(in_file) + file_errors = [] + + try: + # Read input CSV + # Use string type to avoid parsing issues and correctly identify empty strings vs NaNs + df_in = pd.read_csv(in_file, dtype=str) + + # Determine which columns might contain observations. + # A simple heuristic: columns named OBS_VALUE or similar. + # If the dataset has a specific structure, this might need refinement. + obs_columns = [col for col in df_in.columns if col.upper() in ['OBS_VALUE', 'VALUE', 'OBS']] + + if not obs_columns: + self.write_log(f"File {filename}: No clear observation value column found (e.g., OBS_VALUE). Skipping.") + continue + + # For each observation column, iterate over rows + # Pandas iterrows is 0-indexed, meaning row 0 corresponds to line 2 in CSV (line 1 is header). + # But we should use the exact coordinate system used by the generator. + # Typically, `#input` uses format `filename:row:col` (1-based row, 1-based col, or similar). + # Wait, looking at SDG_q1-2026_OBS_AG_FLS_PCT.csv:2:4 + # 2 is the 1-based data row (which is index 0 in pandas, line 2 in the text file). + # Let's parse the exact format from actual_inputs for this file. + + for idx, row in df_in.iterrows(): + csv_row_num = idx + 2 # Header is line 1, first data row is line 2 + + for col_name in obs_columns: + val = row[col_name] + + # Check if it's non-empty (not NaN and not empty string) + if pd.notna(val) and str(val).strip() != "": + # Try to find the coordinate + col_idx = df_in.columns.get_loc(col_name) # 0-based column index + + # Construct expected coordinate + coord = f"{filename}:{csv_row_num}:{col_idx}" + + if coord not in actual_inputs: + file_errors.append(f"Missing mapped input: {coord} (Value: {val})") + total_missing += 1 + + except Exception as e: + file_errors.append(f"Error processing input file {filename}: {e}") + + if file_errors: + failed_files += 1 + self.write_log(f"\nFAILED FILE: {filename}") + for err in file_errors[:20]: + self.write_log(f" - {err}") + if len(file_errors) > 20: + self.write_log(f" ... and {len(file_errors) - 20} more missed observations.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total input files processed: {len(in_csv_files)}") + self.write_log(f"Input files with missing observations: {failed_files}") + self.write_log(f"Total missing observations: {total_missing}") + + status = "PASSED" if failed_files == 0 else "FAILED" + self.write_log(f"Overall Status: {status}") + + print(f"Rule 16 Validation completed. {failed_files}/{len(in_csv_files)} files failed.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser(description="Run Rule 16 Validation (Observation Completeness)") + parser.add_argument("--dataset", required=True, help="Dataset name (e.g., sdg_q1-2026)") + parser.add_argument("--dataset_dir", required=True, help="Path to the dataset directory") + parser.add_argument("--input_data_dir", required=True, help="Path to raw input DATA directory") + args = parser.parse_args() + + validator = Rule16Validator(args.dataset, args.dataset_dir, input_data_dir=args.input_data_dir) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_17.py b/scripts/un/un_dataset_validator/scripts/test_rule_17.py new file mode 100644 index 0000000000..743359a80e --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_17.py @@ -0,0 +1,136 @@ +""" +VALIDATION RULE: +================ +# Rule 17: Name Constraints (Bracket and Code Prohibitions) + +## 1. Rule Description +* **Bracket Check:** A `name` property must not start with a bracket character `[` (ignoring leading whitespace). +* **Code Concept Check:** A `name` property must not contain raw concept codes or attribute codes (such as `CL`, `FSP`, `TFT`, or uppercase tokens from the node's own `dcid`). Only descriptive, human-readable names should be present. + +## 2. Files Involved +* **Input MCF Files:** All MCF files located in the dataset's `schema` directory. +""" + +import os +import sys +import glob +import re + +# Add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) +from base_validator import BaseRuleValidator + +class Rule17Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 17 (Name Constraints)") + + # We dynamically target the schema directory + schema_pattern = os.path.join(self.schema_dir, "*.mcf") + schema_files = glob.glob(schema_pattern) + + failed = False + errors = [] + warnings = [] + + if not schema_files: + self.write_log(f"No MCF files found in schema directory: {self.schema_dir}") + failed = True + else: + self.write_log(f"Processing {len(schema_files)} MCF files in schema directory...") + + # Explicitly prohibited standalone codes + prohibited_codes = {"FSP", "TFT", "DSD", "CL"} + + for schema_file in schema_files: + filename = os.path.basename(schema_file) + self.write_log(f"Validating file: {filename}") + + current_node_id = None + current_node_dcid = None + current_node_type = None + + with open(schema_file, 'r', encoding='utf-8') as f: + for line_idx, line in enumerate(f, start=1): + stripped_line = line.strip() + + # Track current node ID/dcid + if stripped_line.startswith("Node: "): + current_node_id = stripped_line[6:].strip() + # Extract dcid + if current_node_id.startswith("dcid:"): + current_node_dcid = current_node_id[5:] + else: + current_node_dcid = current_node_id + current_node_type = None # Reset type for new node + continue + + if stripped_line.startswith("typeOf: "): + current_node_type = stripped_line[8:].strip() + continue + + if stripped_line.startswith("name: "): + # Skip StatVarGroup names as they are group titles and naturally contain classification codes + if current_node_type and "StatVarGroup" in current_node_type: + continue + + # Extract the raw name string + name_val = stripped_line[6:].strip() + if name_val.startswith('"') and name_val.endswith('"'): + name_val = name_val[1:-1] + name_val = name_val.strip() + + if not name_val: + continue + + # 1. Bracket Check: must not start with '[' + if name_val.startswith('['): + errors.append( + f"{filename}, line {line_idx}: Node '{current_node_dcid}' name starts with a bracket: '{name_val}'" + ) + failed = True + + # 2. Code Concept Check: must not contain raw concept codes or attribute codes + # Tokenize name to check for technical codes (words of letters, digits, underscores) + name_tokens = re.findall(r'\b[A-Za-z0-9_]+\b', name_val) + + for token in name_tokens: + # A token is considered a prohibited technical code if: + # - It is explicitly in our list of prohibited codes (FSP, TFT, DSD, CL) + # - It starts with CL_ or DSD_ (case-insensitive) + # - It contains an underscore, is in ALL CAPS (or starts with a letter and has numbers/underscores), and length >= 3 + is_prohibited = ( + token in prohibited_codes or + token.upper().startswith("CL_") or + token.upper().startswith("DSD_") or + (token.isupper() and '_' in token and len(token) >= 3) + ) + + if is_prohibited: + errors.append( + f"{filename}, line {line_idx}: Node '{current_node_dcid}' name contains technical code '{token}' in: '{name_val}'" + ) + failed = True + break # Only report one code error per name property to avoid log clutter + + self.write_log("\n--- Validation Results ---") + if errors: + self.write_log(f"Found {len(errors)} name constraint violations:") + for err in errors[:50]: + self.write_log(f" - {err}") + if len(errors) > 50: + self.write_log(f" ... and {len(errors) - 50} more errors.") + else: + self.write_log("No name constraint violations found (Rule 17 passed).") + + status = "FAILED" if failed else "PASSED" + self.write_log(f"\nOverall Status: {status}") + print(f"Rule 17 Validation complete. Results written to {self.log_file}") + + return not failed + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_17.py ") + sys.exit(1) + validator = Rule17Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_2.py b/scripts/un/un_dataset_validator/scripts/test_rule_2.py new file mode 100644 index 0000000000..7b5f1fac7e --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_2.py @@ -0,0 +1,148 @@ +""" +VALIDATION RULE: +================ +# Rule 2: SERIES Mapping to `populationType` + +## 1. Rule Description +* **Core Rule (2):** The `series` identifier from the input file must be mapped to the `populationType` property on the generated Statistical Variable nodes in the output `.mcf` file. +* **Sub-rule (2.1):** The series code in the `populationType` must be prefixed with the responsible agency's identifier in the format: `UN__SERIES-`. + +## 2. Files Involved +* **Input Dataset / Context:** The raw input data containing the series information. The series code can typically be derived from the input filename (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv` implies the series `AG_FLS_INDEX`) or from an explicit column within the input CSV. +* **Output MCF File:** The generated StatVars file (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf`). + +## 3. Concrete Example (SDG Dataset) +* **Dataset Context:** `SDG` (derived from the folder name `sdg_q1-2026`). Agency becomes `SDG`. +* **Input Series:** `AG_FLS_INDEX`. +* **Expected Prefix Formation:** `UN_` + `SDG` + `_SERIES-` + `AG_FLS_INDEX` = `UN_SDG_SERIES-AG_FLS_INDEX`. +* **Output Check:** Look at the `.mcf` file. For a node like `Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD`, you must find the exact property line: + ``` + populationType: dcid:UN_SDG_SERIES-AG_FLS_INDEX + ``` + + +""" +import os +import sys +import glob +import random + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator + +def parse_mcf(filepath): + nodes = [] + current_node = {} + with open(filepath, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + if current_node: + nodes.append(current_node) + current_node = {} + continue + if line.startswith('Node:'): + if current_node: + nodes.append(current_node) + current_node = {'Node': line.split(':', 1)[1].strip()} + elif ':' in line: + key, val = line.split(':', 1) + current_node[key.strip()] = val.strip() + if current_node: + nodes.append(current_node) + return nodes + +class Rule2Validator(BaseRuleValidator): + def validate_file(self, mcf_filepath: str, expected_series_code: str, agency_name: str) -> dict: + agency_upper = agency_name.upper() + expected_population_type = f"dcid:UN_{agency_upper}_SERIES-{expected_series_code}" + filename = os.path.basename(mcf_filepath) + + mcf_nodes = parse_mcf(mcf_filepath) + errors = [] + + for node in mcf_nodes: + if node.get('typeOf') == 'dcid:StatisticalVariable': + node_id = node.get('Node') + actual_population_type = node.get('populationType') + + if not actual_population_type: + errors.append(f"File {filename}: Node {node_id} is missing 'populationType'.") + elif actual_population_type != expected_population_type: + errors.append(f"File {filename}: Node {node_id} has incorrect populationType. Expected '{expected_population_type}', got '{actual_population_type}'.") + + if errors: + return {"status": "FAILED", "errors": errors} + return {"status": "PASSED"} + + def validate(self): + self.setup_logging("Rule 2 (Series to Population Type)") + + pattern = os.path.join(self.processed_dir, "*_stat_vars.mcf") + mcf_files = glob.glob(pattern) + + if not mcf_files: + self.write_log(f"No *_stat_vars.mcf files found in {self.processed_dir}") + print(f"No *_stat_vars.mcf files found in {self.processed_dir}. Log written to {self.log_file}") + return False + + total = len(mcf_files) + passed = 0 + failed = 0 + failed_details = [] + + for filepath in mcf_files: + filename = os.path.basename(filepath) + + if "_OBS_" not in filename: + continue + + parts = filename.split("_OBS_") + prefix = parts[0] + agency = prefix.split("_")[0] + + suffix = parts[1] + if not suffix.endswith("_data_stat_vars.mcf"): + continue + + series_code = suffix.replace("_data_stat_vars.mcf", "") + + result = self.validate_file(filepath, series_code, agency) + + if result["status"] == "PASSED": + passed += 1 + else: + failed += 1 + failed_details.append({"filename": filename, "errors": result["errors"]}) + + self.write_log(f"--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No failures detected.") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total nodes failed in this file: {len(errors)}") + + # Sample up to 10 random errors + sampled_errors = random.sample(errors, min(10, len(errors))) + self.write_log(f"Sampled failed nodes (up to 10):") + for err in sampled_errors: + self.write_log(f" - {err}") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + + print(f"Rule 2 Validation complete. Results written to {self.log_file}") + return failed == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_2.py ") + sys.exit(1) + validator = Rule2Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_3.py b/scripts/un/un_dataset_validator/scripts/test_rule_3.py new file mode 100644 index 0000000000..834b1c77d5 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_3.py @@ -0,0 +1,206 @@ +""" +VALIDATION RULE: +================ +# Rule 3: GEOGRAPHY Mapping to `observationAbout` + +## 1. Rule Description +* **Core Rule (3):** The geographical identifiers found in the input dataset's geography columns must be mapped to the `observationAbout` column in the generated output `.csv` file. This mapping is performed using a central Property-Value (PV) map. +* **Sub-rule (3.1):** Any place in the input geography column that represents a geographical entity at the "Country" level or above (e.g., continents, global regions, Earth) *must* be successfully resolved to a Data Commons identifier (DCID). If it cannot be resolved, this failure must be explicitly recorded. + +## 2. Files Involved +* **Input Dataset:** The raw data containing a geography column (e.g., `REF_AREA` or `geo`). +* **Geography PV Map:** `/all_data/pvmap/un_geography_pvmap.csv`. This file acts as the dictionary, translating UN geographical codes to valid DCIDs for output creation. +* **Event Geography CSV:** `/all_data/un_geography.csv`. According to the implementation meeting notes, this file should be referred to for the full geographical hierarchy to determine if a missing/unresolved geography code represents a country-level or higher entity. +* **Output CSV File:** The generated data file (e.g., `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`), which must contain the `observationAbout` column. + +## 3. Concrete Example (SDG Dataset) +* **Input Dataset:** A row in the input CSV has a geography code: `G00000020`. +* **PV Map Lookup:** The script looks up `GEOGRAPHY:G00000020` in `un_geography_pvmap.csv` and finds the mapping to the DCID `dcid:country/AFG`. +* **Output Check:** In the corresponding row of the generated `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`, the `observationAbout` column must contain the value `dcid:country/AFG`. + + +""" +import os +import sys +import glob +import random +import csv + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import get_input_file_path + +def load_pv_map(pv_map_path): + geo_map = {} + if not os.path.exists(pv_map_path): + return geo_map + with open(pv_map_path, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + for row in reader: + if len(row) >= 3: + key_parts = row[0].split(':') + if len(key_parts) == 2 and key_parts[0] == 'GEOGRAPHY': + geo_code = key_parts[1].strip() + dcid = row[2].strip() + geo_map[geo_code] = dcid + return geo_map + +def load_geo_names(geo_names_path): + geo_names = {} + if not os.path.exists(geo_names_path): + return geo_names + with open(geo_names_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + geo_code = row.get('CODE', '') + name = row.get('NAME_EN', '') + if geo_code: + geo_names[geo_code] = name + return geo_names + +class Rule3Validator(BaseRuleValidator): + def validate_file(self, output_csv_path, geo_map, unmapped_geos): + filename = os.path.basename(output_csv_path) + + output_mapping = {} + with open(output_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + if '#input' in row: + parts = row['#input'].split(':') + if len(parts) >= 2: + input_filename = parts[0] + try: + line_num = int(parts[1]) + output_mapping[(input_filename, line_num)] = row.get('observationAbout', '') + except ValueError: + pass + + if not output_mapping: + return {"status": "SKIPPED", "reason": "No #input mapping found in output CSV"} + + input_filename = list(output_mapping.keys())[0][0] + input_csv_path = get_input_file_path(self.dataset_name, input_filename, self.input_data_dir) + + if not input_csv_path or not os.path.exists(input_csv_path): + return {"status": "FAILED", "errors": [f"Input file not found: {input_filename}"]} + + errors = [] + + with open(input_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for line_idx, row in enumerate(reader, start=2): + geo_code = None + if 'GEOGRAPHY' in row: + geo_code = row['GEOGRAPHY'] + elif 'geo' in row: + geo_code = row['geo'] + elif 'REF_AREA' in row: + geo_code = row['REF_AREA'] + elif 'GeoAreaCode' in row: + geo_code = row['GeoAreaCode'] + + if geo_code is None: + continue + + expected_dcid = geo_map.get(geo_code) + + mapping_key = (input_filename, line_idx) + if mapping_key in output_mapping: + actual_dcid = output_mapping[mapping_key] + if expected_dcid and actual_dcid != expected_dcid: + errors.append(f"File {input_filename}, Line {line_idx}: expected observationAbout '{expected_dcid}' for geo '{geo_code}', but got '{actual_dcid}'") + elif not expected_dcid: + unmapped_geos.add(geo_code) + if not actual_dcid.startswith('dcid:'): + errors.append(f"File {input_filename}, Line {line_idx}: unresolved geo '{geo_code}' produced invalid observationAbout '{actual_dcid}' (missing dcid: prefix)") + else: + if not expected_dcid: + unmapped_geos.add(geo_code) + + if errors: + return {"status": "FAILED", "errors": errors} + + return {"status": "PASSED"} + + def validate(self): + self.setup_logging("Rule 3 (Geography Mapping)") + + pv_map_path = os.path.join(self.pvmap_dir, "un_geography_pvmap.csv") + # Global geography names + geo_names_path = os.path.join(os.path.dirname(self.dataset_dir), "un_geography.csv") + + self.write_log(f"Loading PV Map from {pv_map_path}...") + geo_map = load_pv_map(pv_map_path) + self.write_log(f"Loaded {len(geo_map)} geography mappings.") + + self.write_log(f"Loading Geography Names from {geo_names_path}...") + geo_names = load_geo_names(geo_names_path) + + pattern = os.path.join(self.processed_dir, "*_data.csv") + output_files = glob.glob(pattern) + + unmapped_geos = set() + + if not output_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}") + return False + + total = len(output_files) + passed = 0 + failed = 0 + skipped = 0 + failed_details = [] + + for filepath in output_files: + result = self.validate_file(filepath, geo_map, unmapped_geos) + + if result["status"] == "PASSED": + passed += 1 + elif result["status"] == "SKIPPED": + skipped += 1 + else: + failed += 1 + failed_details.append({"filename": os.path.basename(filepath), "errors": result["errors"]}) + + self.write_log(f"--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No validation failures detected (excluding unmapped geos).") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total row mismatch errors in this file: {len(errors)}") + + sampled_errors = random.sample(errors, min(10, len(errors))) + self.write_log(f"Sampled mismatch errors (up to 10):") + for err in sampled_errors: + self.write_log(f" - {err}") + + self.write_log(f"\n--- Unmapped Geographies Report ---") + if not unmapped_geos: + self.write_log("No unmapped geographies found.") + else: + self.write_log(f"Found {len(unmapped_geos)} unique unmapped geographies that were dropped from the output:") + for geo in sorted(unmapped_geos): + name = geo_names.get(geo, "Unknown Name") + self.write_log(f" - {geo}: {name}") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + self.write_log(f"Skipped: {skipped}") + + print(f"Rule 3 Validation complete. Results written to {self.log_file}") + return failed == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_3.py ") + sys.exit(1) + validator = Rule3Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_4.py b/scripts/un/un_dataset_validator/scripts/test_rule_4.py new file mode 100644 index 0000000000..29d077a5aa --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_4.py @@ -0,0 +1,154 @@ +""" +VALIDATION RULE: +================ +# Rule 4: Time Period to observationDate + +## Objective +Ensure that the `TIME_PERIOD` (or `timePeriod`) column in the source data accurately maps to the Data Commons `observationDate` property in the output CSV, adhering to the new conversion requirements for complex formats. + +## File References +- **Input Data File:** Source data file containing the `TIME_PERIOD` column. +- **Output CSV:** Transcoded data file (e.g., `processed_data/*_data.csv`). + + +""" +import os +import sys +import glob +import random +import csv + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import get_input_file_path + +class Rule4Validator(BaseRuleValidator): + def validate_file(self, output_csv_path): + output_mapping = {} + with open(output_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + if 'observationDate' not in reader.fieldnames: + return {"status": "FAILED", "errors": ["Output CSV missing 'observationDate' column."]} + if '#input' not in reader.fieldnames: + return {"status": "SKIPPED", "reason": "No #input mapping found in output CSV"} + + for row in reader: + parts = row['#input'].split(':') + if len(parts) >= 2: + input_filename = parts[0] + try: + line_num = int(parts[1]) + output_mapping[(input_filename, line_num)] = row.get('observationDate', '').strip() + except ValueError: + pass + + if not output_mapping: + return {"status": "SKIPPED", "reason": "No valid #input mapping parsed"} + + input_filename = list(output_mapping.keys())[0][0] + input_csv_path = get_input_file_path(self.dataset_name, input_filename, self.input_data_dir) + + if not input_csv_path or not os.path.exists(input_csv_path): + return {"status": "FAILED", "errors": [f"Input file not found: {input_filename}"]} + + errors = [] + + with open(input_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + time_col = None + for col in reader.fieldnames: + if col and col.upper() == 'TIME_PERIOD': + time_col = col + break + + if not time_col: + return {"status": "FAILED", "errors": [f"Input CSV missing 'TIME_PERIOD' column (case-insensitive). Found: {reader.fieldnames}"]} + + for line_idx, row in enumerate(reader, start=2): + mapping_key = (input_filename, line_idx) + if mapping_key in output_mapping: + actual_obs_date = output_mapping[mapping_key] + input_time = str(row.get(time_col, '')).strip() + + yyyy_format = input_time[:4] if len(input_time) >= 4 else input_time + + if actual_obs_date == yyyy_format: + continue + + if actual_obs_date == input_time: + continue + + if '/' in input_time: + start_date = input_time.split('/')[0] + if actual_obs_date == start_date: + continue + errors.append(f"File {input_filename}, Line {line_idx}: expected '{yyyy_format}', '{start_date}', or '{input_time}', but got '{actual_obs_date}'") + else: + errors.append(f"File {input_filename}, Line {line_idx}: expected '{yyyy_format}' or '{input_time}', but got '{actual_obs_date}'") + + if errors: + return {"status": "FAILED", "errors": errors} + + return {"status": "PASSED"} + + def validate(self): + self.setup_logging("Rule 4 (Time Period Mapping)") + + pattern = os.path.join(self.processed_dir, "*_data.csv") + output_files = glob.glob(pattern) + + if not output_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}") + return False + + total = len(output_files) + passed = 0 + failed = 0 + skipped = 0 + failed_details = [] + + for filepath in output_files: + result = self.validate_file(filepath) + + if result["status"] == "PASSED": + passed += 1 + elif result["status"] == "SKIPPED": + skipped += 1 + else: + failed += 1 + failed_details.append({"filename": os.path.basename(filepath), "errors": result["errors"]}) + + self.write_log(f"--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No validation failures detected.") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total row mismatch errors in this file: {len(errors)}") + + sampled_errors = random.sample(errors, min(10, len(errors))) + self.write_log(f"Sampled mismatch errors (up to 10):") + for err in sampled_errors: + self.write_log(f" - {err}") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + self.write_log(f"Skipped: {skipped}") + + print(f"Rule 4 Validation complete. Results written to {self.log_file}") + return failed == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_4.py ") + sys.exit(1) + validator = Rule4Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_5_7.py b/scripts/un/un_dataset_validator/scripts/test_rule_5_7.py new file mode 100644 index 0000000000..5b50863f1e --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_5_7.py @@ -0,0 +1,191 @@ +""" +VALIDATION RULE: +================ +# Rule 5: OBS_VALUE Mapping to `value` + +## 1. Rule Description +* **Core Rule (5):** The observation value (`OBS_VALUE` column) from the input data must be mapped to the `value` property in the output CSV. +* **Format Check:** As per the meeting notes, observation values are required to map to `value.number`. This means the resulting `value` must be a valid number (`{Number}`). This validation check must be performed using the property-value (PV) map. + +## 2. Files Involved +* **Input Dataset:** The raw input `.csv` data files containing the `OBS_VALUE` column. +* **Output Data File:** The generated output `.csv` file. +* **Property Value Map (PV Map):** Specifically, the `common_pvmap_obs.csv` file which defines the mapping rule: `OBS_VALUE,value,{Number},...` + +## 3. Concrete Example +* **Input Data Row:** Contains an `OBS_VALUE` of `594982.4482`. +* **Output Check:** The output `.csv` file should contain a column named `value`, and the corresponding row must contain `594982.4482` (or potentially a multiplied value if unit multipliers apply, see Rule 10, but inherently it must be numeric). + + +# Rule 7: UNIT_MULTIPLIER Application + +## Context from Meeting Notes +During the meeting, Harish Chandrashekar explicitly stated: "Wherever unit multiplier is there it whatever multiplication factor it has it should be multiplied with the value. Every multiplier should be multiplied with H value." (00:24:26) + +This means that if a dataset contains a unit multiplier column (e.g., indicating the values are in thousands or millions), the raw observation value (`OBS_VALUE`) must be multiplied by this factor before being written to the output Data Commons CSV. + +## Mapping Logic +1. **Identify Multiplier Attribute:** Identify the column representing the unit multiplier in the raw input data (often named `UNIT_MULT`, `UNIT_MULTIPLIER`, etc., depending on the DSD). +2. **Lookup Factor:** Use the common PV map `all_data/pvmap/CL_MULT_pvmap_multiply.csv` to resolve the multiplication factor. The PV map links the multiplier code to a float value (e.g., `1.00E-15` for -15). +3. **Apply Factor:** The expected value in the Data Commons output CSV should be: + `Expected_Value = float(Raw_OBS_VALUE) * float(Multiplier_Factor)` +4. **Compare:** Assert that `Expected_Value` matches the `value` column in the processed output CSV. + +## Relevant Files +- **Raw Input Data:** Varies per dataset (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv`). +- **PV Map for Multipliers:** `all_data/pvmap/CL_MULT_pvmap_multiply.csv`. +- **Processed Output Data:** Varies per dataset (e.g., `SDG_q1-2026_OBS_AG_FOOD_WST_data.csv` in `processed_data/`). + + +""" +import os +import sys +import glob +import csv +import math + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import load_multipliers, get_input_file_path + +class Rule5And7Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 5 & 7 (OBS_VALUE mapping & Unit Multipliers)") + + multipliers = load_multipliers(self.pvmap_dir) + + pattern = os.path.join(self.processed_dir, "*_data.csv") + output_files = glob.glob(pattern) + + if not output_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}") + return False + + total_files = len(output_files) + total_rows_checked = 0 + total_errors = 0 + file_errors_map = {} + + input_file_cache = {} + + for filepath in output_files: + filename = os.path.basename(filepath) + file_errors = [] + + with open(filepath, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + if 'value' not in reader.fieldnames or '#input' not in reader.fieldnames: + file_errors.append(f"Missing required columns ('value' or '#input') in output.") + file_errors_map[filename] = file_errors + total_errors += 1 + continue + + for output_row_num, row in enumerate(reader, start=2): + input_lineage = row.get('#input', '') + if not input_lineage: + continue + + parts = input_lineage.split(':') + if len(parts) < 2: + continue + + input_filename = parts[0] + try: + input_row_idx = int(parts[1]) + except ValueError: + continue + + if input_filename not in input_file_cache: + in_path = get_input_file_path(self.dataset_name, input_filename, self.input_data_dir) + if not in_path: + file_errors.append(f"Could not find input file: {input_filename}") + input_file_cache[input_filename] = None + continue + + rows_dict = {} + try: + with open(in_path, 'r', encoding='utf-8') as in_f: + in_reader = csv.DictReader(in_f) + for idx, in_row in enumerate(in_reader, start=2): + rows_dict[idx] = in_row + input_file_cache[input_filename] = rows_dict + except Exception as e: + file_errors.append(f"Failed to read input file {input_filename}: {e}") + input_file_cache[input_filename] = None + + input_data = input_file_cache.get(input_filename) + if not input_data: + continue + + in_row = input_data.get(input_row_idx) + if not in_row: + file_errors.append(f"Row {input_row_idx} not found in input file {input_filename}") + continue + + obs_value_str = in_row.get('OBS_VALUE') + unit_mult_code = in_row.get('UNIT_MULT') + output_value_str = row.get('value') + + if obs_value_str is None or output_value_str is None: + continue + + total_rows_checked += 1 + + try: + out_val = float(output_value_str) + except ValueError: + file_errors.append(f"Output row {output_row_num}: 'value' is not numeric ('{output_value_str}')") + continue + + try: + clean_obs = obs_value_str.replace(',', '') + in_val = float(clean_obs) + except ValueError: + file_errors.append(f"Input row {input_row_idx}: OBS_VALUE is not numeric ('{obs_value_str}') but output is '{out_val}'") + continue + + mult = 1.0 + if unit_mult_code and str(unit_mult_code) in multipliers: + mult = multipliers[str(unit_mult_code)] + + expected_val = in_val * mult + + if not math.isclose(expected_val, out_val, rel_tol=1e-4, abs_tol=1e-4): + file_errors.append(f"Row {output_row_num} lineage {input_lineage}: Value mismatch. Expected {in_val} * {mult} = {expected_val}, got {out_val}") + + if file_errors: + file_errors_map[filename] = file_errors + total_errors += len(file_errors) + + self.write_log(f"--- File-Specific Violations ---") + if not file_errors_map: + self.write_log("No violations detected.") + else: + for filename, errors in file_errors_map.items(): + self.write_log(f"\nFAILED FILE: {filename}") + for err in errors[:20]: + self.write_log(f" - {err}") + if len(errors) > 20: + self.write_log(f" ... and {len(errors) - 20} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total output files processed: {total_files}") + self.write_log(f"Total rows checked: {total_rows_checked}") + self.write_log(f"Total errors: {total_errors}") + + status = "PASSED" if total_errors == 0 else "FAILED" + self.write_log(f"Overall Status: {status}") + + print(f"Rule 5 & 7 Validation complete. Checked {total_rows_checked} rows. Results written to {self.log_file}") + return total_errors == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_5_7.py ") + sys.exit(1) + validator = Rule5And7Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_6.py b/scripts/un/un_dataset_validator/scripts/test_rule_6.py new file mode 100644 index 0000000000..b6b8d5b5d7 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_6.py @@ -0,0 +1,236 @@ +""" +VALIDATION RULE: +================ +# Rule 6: Dimensions vs. Attributes + +## 1. Rule Description +* **Core Rule (6):** All properties defined in the Dataset Definition (DSD) file must be correctly handled as either a **dimension** or an **attribute**. +* **Dimensions:** Except for specific exemptions (`Geography`, `Time Period`, `OBS_VALUE`, and `SERIES`), any property marked as a "dimension" must be attached to the Statistical Variable (StatVar) as a **constraint property**. +* **Attributes:** Any property marked as an "attribute" must **not** be attached to the Statistical Variable. Instead, it must be included in the output data `.csv` as a separate column. + +## 2. Files Involved +* **Input DSD File:** The respective DSD file for the dataset (e.g., an Excel/CSV file containing the `ROLE` column with values "dimension" or "attribute"). +* **Input Data File:** The raw `.csv` data. +* **Output Data File:** The generated output `.csv` file. +* **Output MCF File:** The generated `.mcf` file containing the Statistical Variable definitions. + +## 3. Concrete Example +* **DSD Definition:** + * `PRODUCT` is defined with a `ROLE` of "dimension". + * `CENSORED_VALUE_TYPE` is defined with a `ROLE` of "attribute". +* **Output Check for Dimension (`PRODUCT`):** + * The Statistical Variable node (e.g., `Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_CRL_PUL`) in the `.mcf` file must contain `PRODUCT` as a constraint property (e.g., `product: dcid:UN_PRODUCT-AGG_CRL_PUL`). +* **Output Check for Attribute (`CENSORED_VALUE_TYPE`):** + * The StatVar node must **not** contain `CENSORED_VALUE_TYPE`. + * The output `.csv` file must contain a separate column named `censoredValueType` (or similar mapped name), with values like `UN_CENSORED_VALUE_TYPE-_Z`. + + +# Rule 6.1: Dimension Mapping to DCP Schema with UN_ Prefix + +## Requirement +Each dimension must be mapped to the Data Commons Project (DCP) schema using a common prefix `UN_` without including the specific agency prefix. + +## Context & Rules (From Meeting Notes) +- The dataset is processed to create a schema. Any schema generated from the DCP must consistently use the `UN_` prefix for dimensions. +- Every node within the agency-specific schema files follows a "concept_value" structure. +- The mapping must ensure these dimensions correctly represent the concept without introducing agency-specific identifiers in the prefix (e.g., use `UN_` instead of `UN_ECLAC_`). + + +# Rule 6.2: Dimension Value Mapping + +## Objective +Ensure that every value within a dimension column is properly mapped to a generated property value DCID following the exact template: `_-`. + +## Context & Rationale +According to the meeting notes and project requirements, any column designated as a "dimension" in the Dataset Definition (DSD) file must have its distinct values transformed into standardized Data Commons Identifiers (DCIDs). The format for these DCIDs consistently incorporates a prefix (typically `UN`), the concept name (derived from the column name), and the specific value code. + +For instance, if the dimension column is `product` and a raw value in the dataset is `CPC2_1_0113`, the mapped value DCID should be formatted as `UN_PRODUCT-CPC2_1_0113`. In the output schema/MCF, this would appear as a property-value pair like `product: dcid:UN_PRODUCT-CPC2_1_0113`. + +## Files Involved +* **DSD File (e.g., `schema.csv` or `DSD.csv`):** Used to identify which columns act as dimensions (where `ROLE` = `Dimension`). +* **Input Data File (`.csv`):** Provides the raw source values for the identified dimension columns. +* **Output Files (`.mcf`, `.tmcf`, or processed `.csv`):** Validated to ensure the final output respects the mapped DCID format. + +## Verification Logic +1. **Identify Dimensions:** Read the DSD file and filter for rows where the `ROLE` column is explicitly set to `Dimension` (excluding explicit exceptions like Geography, Time Period, OBS_VALUE, and Series, which have their own rules). +2. **Extract Concept Names:** Determine the concept name from the dimension's column name. The concept name is generally the uppercase version of the column name (e.g., `product` -> `PRODUCT`). +3. **Construct Expected DCIDs:** For every row in the input data file, look at the value under the dimension column. Construct the expected DCID using the template: + * `Prefix`: `UN` (or derived from the specific dataset configuration). + * `Concept`: Capitalized column name. + * `Code`: The raw value found in the input data. + * **Format:** `_-` (e.g., `UN_PRODUCT-CPC2_1_0113`). +4. **Validate Output:** Ensure that in the finalized statistical variable mapping or output nodes, the dimension property maps exactly to this constructed DCID (e.g., checking that `product: dcid:UN_PRODUCT-CPC2_1_0113` exists). + + +""" +import os +import sys +import glob +import csv +import re + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator + +def sanitize_for_match(text: str) -> str: + """Removes underscores and converts to lowercase for column matching.""" + return text.replace("_", "").lower() + +class Rule6Validator(BaseRuleValidator): + def validate(self): + self.setup_logging("Rule 6 (Dimension vs Attribute Mapping)") + + if not self.dsd_dir or not os.path.exists(self.dsd_dir): + self.write_log(f"Could not find DSD directory for dataset {self.dataset_name}") + print(f"Could not find DSD directory for dataset {self.dataset_name}") + return False + + dsd_dir = self.dsd_dir + dsd_files = glob.glob(os.path.join(dsd_dir, "*.csv")) + + if not dsd_files: + self.write_log(f"No DSD files found in {dsd_dir}") + return False + + exemptions = ['SERIES', 'GEOGRAPHY', 'TIME_PERIOD', 'OBS_VALUE'] + total_files = len(dsd_files) + failed_files = 0 + + self.write_log(f"DSD Directory: {dsd_dir}") + self.write_log(f"Output Directory: {self.processed_dir}\n") + + for dsd_path in sorted(dsd_files): + filename = os.path.basename(dsd_path) + match = re.search(r'_DSD_(.*?)\.csv', filename) + if not match: + continue + series = match.group(1) + + mcf_pattern = os.path.join(self.processed_dir, f"*_OBS_{series}_data_stat_vars.mcf") + csv_pattern = os.path.join(self.processed_dir, f"*_OBS_{series}_data.csv") + + mcf_files = glob.glob(mcf_pattern) + csv_files = glob.glob(csv_pattern) + + if not mcf_files or not csv_files: + self.write_log(f"[{series}] SKIPPED: Missing corresponding MCF or CSV file.") + continue + + mcf_path = mcf_files[0] + csv_path = csv_files[0] + + errors = [] + + dimensions = [] + attributes = [] + with open(dsd_path, 'r', encoding='utf-8-sig') as f: + reader = csv.DictReader(f) + for row in reader: + role = row.get('ROLE', '').strip().lower() + concept = row.get('CONCEPT', '').strip() + + if role == 'dimension': + if concept.upper() not in exemptions: + dimensions.append(concept) + elif role == 'attribute': + attributes.append(concept) + + with open(mcf_path, 'r', encoding='utf-8') as f: + mcf_content = f.read() + + with open(csv_path, 'r', encoding='utf-8') as f: + csv_reader = csv.reader(f) + headers = next(csv_reader, []) + sanitized_headers = [sanitize_for_match(h) for h in headers] + + dim_indices = [] + dim_col_names = [] + for dim in dimensions: + expected_col = sanitize_for_match(dim) + if expected_col in sanitized_headers: + dim_indices.append(sanitized_headers.index(expected_col)) + dim_col_names.append(dim) + + dim_values = {dim: set() for dim in dim_col_names} + if dim_indices: + for row in csv_reader: + for idx, dim in zip(dim_indices, dim_col_names): + if idx < len(row): + val = row[idx].strip() + if val: + dim_values[dim].add(val) + + for dim in dimensions: + dim_upper = dim.upper() + + # Rule 6.1: strict UN_ prefix mapping + expected_prefix = f"dcid:UN_{dim_upper}-" + if expected_prefix not in mcf_content: + # Look for agency-specific prefix violation + match = re.search(rf"dcid:(UN_[A-Z0-9]+_{dim_upper}-)", mcf_content) + if match: + errors.append(f"File {os.path.basename(mcf_path)}: Rule 6.1 Violation - Dimension '{dim}' uses an agency-specific prefix '{match.group(1)}' instead of 'UN_{dim_upper}-'.") + else: + # It might be present without UN_ or completely missing + if f"_{dim_upper}-" in mcf_content: + errors.append(f"File {os.path.basename(mcf_path)}: Rule 6.1 Violation - Dimension '{dim}' found but without the required 'UN_' prefix.") + else: + errors.append(f"File {os.path.basename(mcf_path)}: Dimension '{dim}' not found as a constraint property in MCF.") + + # Rule 6.2: Dimension Value Mapping + if dim in dim_values: + for val in dim_values[dim]: + clean_val = re.sub(r'[^a-zA-Z0-9_]', '_', str(val)) + expected_dcid = f"UN_{dim_upper}-{clean_val}" + if expected_dcid not in mcf_content: + errors.append(f"File {os.path.basename(mcf_path)}: Rule 6.2 Violation - Dimension '{dim}' value '{val}' missing expected mapped DCID '{expected_dcid}'.") + + attr_column_map = { + 'UNIT_MEASURE': 'unit', + 'FREQUENCY': 'opservationperiod', # Pipeline typo + 'UNIT_MULT': None, # Consumed by Rule 7, not in CSV + } + + for attr in attributes: + attr_upper = attr.upper() + + if attr_upper == 'UNIT_MULT' or (attr_column_map.get(attr_upper) is None and attr_upper in attr_column_map): + continue + + attr_prop_name = attr.lower() + if re.search(rf'^{attr_prop_name}:\s*dcid:', mcf_content, re.MULTILINE | re.IGNORECASE): + errors.append(f"File {os.path.basename(mcf_path)}: Attribute '{attr}' incorrectly attached to a StatVar in the MCF file.") + if f"UN_{attr_upper}-" in mcf_content: + errors.append(f"File {os.path.basename(mcf_path)}: Attribute '{attr}' found with dimension-like DCID 'UN_{attr_upper}-' in MCF.") + + expected_col = attr_column_map.get(attr_upper, sanitize_for_match(attr)) + if expected_col not in sanitized_headers: + errors.append(f"File {os.path.basename(csv_path)}: Attribute '{attr}' not found as a separate column in the output CSV. Expected a column matching '{expected_col}'.") + + if errors: + failed_files += 1 + self.write_log(f"[{series}] FAILED") + for err in errors: + self.write_log(f" - {err}") + self.write_log("") + else: + self.write_log(f"[{series}] PASSED") + + self.write_log(f"\nSummary:") + self.write_log(f"Total DSDs Checked: {total_files}") + self.write_log(f"Passed: {total_files - failed_files}") + self.write_log(f"Failed: {failed_files}") + + print(f"Rule 6 Validation completed. {failed_files}/{total_files} files failed.") + print(f"Details saved to {self.log_file}") + + return failed_files == 0 + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_6.py ") + sys.exit(1) + validator = Rule6Validator(sys.argv[1], sys.argv[2]) + validator.validate() \ No newline at end of file diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_8.py b/scripts/un/un_dataset_validator/scripts/test_rule_8.py new file mode 100644 index 0000000000..268f56b1f0 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_8.py @@ -0,0 +1,226 @@ +""" +VALIDATION RULE: +================ +# Rule 8: UNIT_MEASURE Mapping and Multiplier Logic Integration + +## Requirement +`UNIT_MEASURE` must be mapped to a Data Commons Project (DCP) enum using the name from the source dataset. Additionally, validation logic for `UNIT_MEASURE` should be closely integrated with multiplier logic. + +## Context & Rules (From Meeting Notes) +- The checklist initially marked this as an "ASK AJAI" item regarding how to map `UNIT_MEASURE` to a DCP enum and what to set in the `shortDisplayName`. +- During the meeting, it was agreed that the mapping for rule number eight involves similar multiplier logic to what is used for applying multipliers (Rule 7). +- The decision was finalized to integrate this mapping and the associated multiplier logic into the existing validation code to ensure consistency when validating how units and multipliers are applied to values. + + +""" +import os +import sys +import glob +import csv + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import get_input_file_path + +def load_unit_measure_map(pvmap_dir: str) -> dict: + """Load UNIT_MEASURE -> DCP enum mappings from the PV map file.""" + candidates = [ + os.path.join(pvmap_dir, "CL_UNIT_MEASURE_pvmap.csv"), + os.path.join(pvmap_dir, "common_pvmap_obs.csv"), + ] + # Also search global pvmap directory + global_pvmap = os.path.join( + os.path.dirname(os.path.dirname(pvmap_dir)), "pvmap" + ) + candidates += [ + os.path.join(global_pvmap, "CL_UNIT_MEASURE_pvmap.csv"), + os.path.join(global_pvmap, "common_pvmap_obs.csv"), + ] + + for path in candidates: + if not os.path.exists(path): + continue + unit_map = {} + with open(path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + # Support two common PV map schemas + un_code = (row.get('UnCode') or row.get('UnCodeKey') or '').strip().strip('"') + prop_val = (row.get('ConstraintPropValue') or row.get('val') or '').strip() + prop = (row.get('prop') or row.get('ConstraintProp') or '').strip() + + # Only capture rows that map UNIT_MEASURE codes (not multipliers) + if un_code and prop_val and 'MULT' not in un_code.upper() and 'FREQ' not in un_code.upper(): + unit_map[un_code] = prop_val + if unit_map: + return unit_map + + return {} + + +class Rule8Validator(BaseRuleValidator): + def validate_file(self, output_csv_path: str, unit_map: dict) -> dict: + output_mapping = {} # (input_filename, line_num) -> unit_value + typo_col = None + + with open(output_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + if '#input' not in (reader.fieldnames or []): + return {"status": "SKIPPED", "reason": "No #input column in output CSV"} + + # Locate the unit column (allow for casing variants) + unit_col = next( + (c for c in (reader.fieldnames or []) if c.lower() == 'unit'), + None + ) + if not unit_col: + return {"status": "FAILED", "errors": ["Output CSV missing 'unit' column for UNIT_MEASURE."]} + + for row in reader: + parts = row.get('#input', '').split(':') + if len(parts) >= 2: + try: + line_num = int(parts[1]) + output_mapping[(parts[0], line_num)] = row.get(unit_col, '').strip() + except ValueError: + pass + + if not output_mapping: + return {"status": "SKIPPED", "reason": "No valid #input rows parsed"} + + input_filename = list(output_mapping.keys())[0][0] + input_csv_path = get_input_file_path(self.dataset_name, input_filename, self.input_data_dir) + if not input_csv_path or not os.path.exists(input_csv_path): + return {"status": "FAILED", "errors": [f"Input file not found: {input_filename}"]} + + errors = [] + unmapped_codes = set() + + with open(input_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + fields_upper = {c.upper(): c for c in (reader.fieldnames or [])} + + unit_measure_col = ( + fields_upper.get('UNIT_MEASURE') or + fields_upper.get('UNIT_MSR') or + fields_upper.get('MEASURE') + ) + if not unit_measure_col: + return {"status": "SKIPPED", "reason": "No UNIT_MEASURE column in input CSV"} + + for line_idx, row in enumerate(reader, start=2): + raw_code = row.get(unit_measure_col, '').strip() + if not raw_code: + continue + + mapping_key = (input_filename, line_idx) + if mapping_key not in output_mapping: + continue + + actual_unit = output_mapping[mapping_key] + + if not unit_map: + # No PV map loaded — just verify the unit column is non-empty + if not actual_unit: + errors.append( + f"File {input_filename}, Line {line_idx}: UNIT_MEASURE '{raw_code}' present in input " + f"but 'unit' column is empty in output." + ) + continue + + expected_unit = unit_map.get(raw_code) + if not expected_unit: + unmapped_codes.add(raw_code) + continue + + if actual_unit != expected_unit: + errors.append( + f"File {input_filename}, Line {line_idx}: UNIT_MEASURE '{raw_code}' -> " + f"expected '{expected_unit}', got '{actual_unit}'." + ) + + if unmapped_codes: + errors.append( + f"UNIT_MEASURE codes not found in PV map (no expected value to verify): " + f"{sorted(unmapped_codes)}" + ) + + if errors: + return {"status": "FAILED", "errors": errors} + return {"status": "PASSED"} + + def validate(self): + self.setup_logging("Rule 8 (UNIT_MEASURE Mapping)") + + unit_map = load_unit_measure_map(self.pvmap_dir) + if not unit_map: + self.write_log( + "WARNING: No UNIT_MEASURE PV map found. Validation will only check " + "that the 'unit' column is non-empty when UNIT_MEASURE is present." + ) + else: + self.write_log(f"Loaded {len(unit_map)} UNIT_MEASURE mappings.") + + pattern = os.path.join(self.processed_dir, "*_data.csv") + output_files = glob.glob(pattern) + + if not output_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}") + return False + + total = len(output_files) + passed = 0 + failed = 0 + skipped = 0 + failed_details = [] + + for filepath in output_files: + result = self.validate_file(filepath, unit_map) + + if result["status"] == "PASSED": + passed += 1 + elif result["status"] == "SKIPPED": + skipped += 1 + self.write_log(f"SKIPPED: {os.path.basename(filepath)} — {result.get('reason', '')}") + else: + failed += 1 + failed_details.append({ + "filename": os.path.basename(filepath), + "errors": result["errors"], + }) + + self.write_log(f"\n--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No validation failures detected.") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total errors: {len(errors)}") + for err in errors[:20]: + self.write_log(f" - {err}") + if len(errors) > 20: + self.write_log(f" ... and {len(errors) - 20} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + self.write_log(f"Skipped: {skipped}") + + status_passed = failed == 0 + self.write_log(f"Overall Status: {'PASSED' if status_passed else 'FAILED'}") + print(f"Rule 8 Validation complete. Results written to {self.log_file}") + return status_passed + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_8.py ") + sys.exit(1) + validator = Rule8Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/test_rule_9.py b/scripts/un/un_dataset_validator/scripts/test_rule_9.py new file mode 100644 index 0000000000..3376d0c640 --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/test_rule_9.py @@ -0,0 +1,193 @@ +""" +VALIDATION RULE: +================ +# Rule 9: Frequency to observationPeriod + +## Objective +Ensure that the `FREQUENCY` column in the source data correctly maps to the Data Commons `observationPeriod` property in the output CSV, adhering to the mapping defined in the common Property-Value (PV) map. + +## File References +- **Input Data File:** Data file containing the `FREQUENCY` column. +- **Common PV Map:** `all_data/pvmap/CL_FREQUENCY_pvmap_obsperiod.csv` +- **Output CSV:** Transcoded data file (e.g., `processed_data/*_data.csv`). + + +""" +import os +import sys +import glob +import csv + +# We add the current directory to sys.path so we can import modules +sys.path.append(os.path.dirname(__file__)) + +from base_validator import BaseRuleValidator +from validator_utils import get_input_file_path + +def load_freq_map(pv_map_path): + freq_map = {} + if not os.path.exists(pv_map_path): + return freq_map + with open(pv_map_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + un_code = row.get('UnCode', '').strip().strip('"') + obs_period = row.get('ConstraintPropValue', '').strip() + if un_code and obs_period: + freq_map[un_code] = obs_period + return freq_map + +class Rule9Validator(BaseRuleValidator): + def validate_file(self, output_csv_path, freq_map): + filename = os.path.basename(output_csv_path) + + output_mapping = {} + target_col = 'observationPeriod' + typo_found = False + + with open(output_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + + if 'observationPeriod' not in reader.fieldnames: + if 'opservationPeriod' in reader.fieldnames: + typo_found = True + target_col = 'opservationPeriod' + else: + # Missing both + return {"status": "FAILED", "errors": ["Target column 'observationPeriod' not found in output CSV."]} + + for row in reader: + if '#input' in row: + parts = row['#input'].split(':') + if len(parts) >= 2: + input_filename = parts[0] + try: + line_num = int(parts[1]) + output_mapping[(input_filename, line_num)] = row.get(target_col, '').strip() + except ValueError: + pass + + if not output_mapping: + return {"status": "SKIPPED", "reason": "No #input mapping found in output CSV"} + + input_filename = list(output_mapping.keys())[0][0] + input_csv_path = get_input_file_path(self.dataset_name, input_filename, self.input_data_dir) + + if not input_csv_path or not os.path.exists(input_csv_path): + return {"status": "FAILED", "errors": [f"Input file not found: {input_filename}"]} + + errors = [] + if typo_found: + errors.append("Typo found in output CSV header: 'opservationPeriod' instead of 'observationPeriod'.") + + with open(input_csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + # FREQUENCY might not be present in all files, but if it is, validate it + if 'FREQUENCY' not in reader.fieldnames and 'FREQ' not in reader.fieldnames: + return {"status": "SKIPPED", "reason": "No FREQUENCY column in input CSV"} + + freq_col = 'FREQUENCY' if 'FREQUENCY' in reader.fieldnames else 'FREQ' + + for line_idx, row in enumerate(reader, start=2): + input_freq = row.get(freq_col, '').strip() + if not input_freq: + continue + + expected_obs_period = freq_map.get(input_freq) + if not expected_obs_period: + continue + + mapping_key = (input_filename, line_idx) + if mapping_key in output_mapping: + actual_obs_period = output_mapping[mapping_key] + if actual_obs_period != expected_obs_period: + errors.append(f"File {input_filename}, Line {line_idx}: expected observationPeriod '{expected_obs_period}' for frequency '{input_freq}', but got '{actual_obs_period}'") + + if errors: + return {"status": "FAILED", "errors": errors} + + return {"status": "PASSED"} + + def validate(self): + self.setup_logging("Rule 9 (Frequency to observationPeriod Mapping)") + + # global pvmap directory + # Use the parent directory of the provided dataset directory to find the global pvmap + # E.g. if dataset_dir is /path/to/data/dataset_name, global pvmap should be /path/to/data/pvmap + global_pvmap_dir = os.path.join(os.path.dirname(self.dataset_dir), "pvmap") + + # Check local dataset pvmap dir first, then global + pv_map_path = os.path.join(self.pvmap_dir, "CL_FREQUENCY_pvmap_obsperiod.csv") + if not os.path.exists(pv_map_path): + pv_map_path = os.path.join(self.pvmap_dir, "common_pvmap_obs.csv") + if not os.path.exists(pv_map_path): + pv_map_path = os.path.join(global_pvmap_dir, "CL_FREQUENCY_pvmap_obsperiod.csv") + if not os.path.exists(pv_map_path): + pv_map_path = os.path.join(global_pvmap_dir, "common_pvmap_obs.csv") + + self.write_log(f"Loading Frequency PV Map from {pv_map_path}...") + freq_map = load_freq_map(pv_map_path) + self.write_log(f"Loaded {len(freq_map)} frequency mappings.") + + if not freq_map: + self.write_log(f"Failed to load frequency mapping from {pv_map_path}") + print(f"Failed to load frequency mapping from {pv_map_path}. Log written to {self.log_file}") + return False + + pattern = os.path.join(self.processed_dir, "*_data.csv") + output_files = glob.glob(pattern) + + if not output_files: + self.write_log(f"No *_data.csv files found in {self.processed_dir}") + print(f"No *_data.csv files found in {self.processed_dir}. Log written to {self.log_file}") + return False + + total = len(output_files) + passed = 0 + failed = 0 + skipped = 0 + failed_details = [] + + for filepath in output_files: + result = self.validate_file(filepath, freq_map) + + if result["status"] == "PASSED": + passed += 1 + elif result["status"] == "SKIPPED": + skipped += 1 + else: + failed += 1 + failed_details.append({"filename": os.path.basename(filepath), "errors": result["errors"]}) + + self.write_log(f"--- Detailed Failure Report ---") + if failed == 0: + self.write_log("No validation failures detected.") + else: + for failure in failed_details: + self.write_log(f"\nFAILED FILE: {failure['filename']}") + errors = failure['errors'] + self.write_log(f"Total errors in this file: {len(errors)}") + + limit = min(10, len(errors)) + for err in errors[:limit]: + self.write_log(f" - {err}") + if len(errors) > limit: + self.write_log(f" ... and {len(errors) - limit} more errors.") + + self.write_log(f"\n--- Validation Summary ---") + self.write_log(f"Total files processed: {total}") + self.write_log(f"Passed: {passed}") + self.write_log(f"Failed: {failed}") + self.write_log(f"Skipped: {skipped}") + + status_passed = failed == 0 + self.write_log(f"Overall Status: {'PASSED' if status_passed else 'FAILED'}") + print(f"Rule 9 Validation complete. Results written to {self.log_file}") + return status_passed + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python test_rule_9.py ") + sys.exit(1) + validator = Rule9Validator(sys.argv[1], sys.argv[2]) + validator.validate() diff --git a/scripts/un/un_dataset_validator/scripts/validator_utils.py b/scripts/un/un_dataset_validator/scripts/validator_utils.py new file mode 100644 index 0000000000..080b7dffac --- /dev/null +++ b/scripts/un/un_dataset_validator/scripts/validator_utils.py @@ -0,0 +1,51 @@ +import os +import glob +import csv +import re + +def to_canonical_format(text: str) -> str: + """Removes quotes, spaces, underscores, and converts to lowercase for alignment matching.""" + return re.sub(r'[^a-z0-9]', '', text.strip('"').lower()) + +def load_multipliers(pvmap_dir: str) -> dict: + """Loads the UNIT_MULT multipliers from the PV map.""" + mult_file = os.path.join(pvmap_dir, "CL_MULT_pvmap_multiply.csv") + + # If not found in agency pvmap, try global pvmap (for ILO it might be there, or SDG) + if not os.path.exists(mult_file): + # Look in the parent directory's pvmap + global_pvmap = os.path.join(os.path.dirname(os.path.dirname(pvmap_dir)), "pvmap", "CL_MULT_pvmap_multiply.csv") + if os.path.exists(global_pvmap): + mult_file = global_pvmap + else: + return {} # No multiplier file + + multipliers = {} + with open(mult_file, 'r', encoding='utf-8') as f: + reader = csv.reader(f) + for row in reader: + if not row or row[0].startswith('#') or row[0] == 'UnCodeKey': + continue + # format: UNIT_MULT:-15,#Multiply,1.00E-15 + if len(row) >= 3 and row[0].startswith("UNIT_MULT:"): + key = row[0].split(":", 1)[1] + try: + multipliers[key] = float(row[2]) + except ValueError: + continue + return multipliers + +def get_input_file_path(dataset_name: str, filename: str, input_data_dir: str) -> str: + """Finds the absolute path to the raw input DATA file.""" + if not input_data_dir: + return None + full_path = os.path.join(input_data_dir, filename) + if os.path.exists(full_path): + return full_path + + # Check inside DATA subdirectory + data_path = os.path.join(input_data_dir, "DATA", filename) + if os.path.exists(data_path): + return data_path + + return None diff --git a/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data.csv b/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data.csv new file mode 100644 index 0000000000..44275d2e9c --- /dev/null +++ b/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data.csv @@ -0,0 +1,150 @@ +observationAbout,observationDate,value,variableMeasured,unit,opservationPeriod,basePeriod,censoredValueType,foodWasteSector,footnote,lowerBound,nature,observationStatus,serviceAttribute,source,#input +dcid:Earth,2015,8.5,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:2:4 +dcid:Earth,2015,12.6,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:3:4 +dcid:Earth,2015,13,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:4:4 +dcid:Earth,2015,13.9,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:5:4 +dcid:Earth,2015,23.2,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:6:4 +dcid:Earth,2016,8.2,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:7:4 +dcid:Earth,2016,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:8:4 +dcid:Earth,2016,12.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:9:4 +dcid:Earth,2016,13.7,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:10:4 +dcid:Earth,2016,22.8,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:11:4 +dcid:Earth,2017,8.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:12:4 +dcid:Earth,2017,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:13:4 +dcid:Earth,2017,12.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:14:4 +dcid:Earth,2017,13.8,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:15:4 +dcid:Earth,2017,22.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:16:4 +dcid:Earth,2018,8.5,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:17:4 +dcid:Earth,2018,12.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:18:4 +dcid:Earth,2018,13.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:19:4 +dcid:Earth,2018,13.9,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:20:4 +dcid:Earth,2018,24.1,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:21:4 +dcid:Earth,2019,8.5,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:22:4 +dcid:Earth,2019,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:23:4 +dcid:Earth,2019,13.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:24:4 +dcid:Earth,2019,14,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:25:4 +dcid:Earth,2019,25.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:26:4 +dcid:Earth,2020,8.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:27:4 +dcid:Earth,2020,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:28:4 +dcid:Earth,2020,13.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:29:4 +dcid:Earth,2020,14,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:30:4 +dcid:Earth,2020,25.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:31:4 +dcid:Earth,2021,8.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:32:4 +dcid:Earth,2021,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:33:4 +dcid:Earth,2021,13.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:34:4 +dcid:Earth,2021,14,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:35:4 +dcid:Earth,2021,25.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:36:4 +dcid:Earth,2022,8.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:37:4 +dcid:Earth,2022,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:38:4 +dcid:Earth,2022,13.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:39:4 +dcid:Earth,2022,14,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:40:4 +dcid:Earth,2022,25.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:41:4 +dcid:Earth,2023,8.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:42:4 +dcid:Earth,2023,12.3,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:43:4 +dcid:Earth,2023,13.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:44:4 +dcid:Earth,2023,14,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:45:4 +dcid:Earth,2023,25.4,dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:46:4 +dcid:NorthernAfrica,2015,16.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:47:4 +dcid:NorthernAfrica,2016,14.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:48:4 +dcid:NorthernAfrica,2017,15.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:49:4 +dcid:NorthernAfrica,2018,15.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:50:4 +dcid:NorthernAfrica,2019,15.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:51:4 +dcid:NorthernAfrica,2020,16,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:52:4 +dcid:NorthernAfrica,2021,16,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:53:4 +dcid:NorthernAfrica,2022,16,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:54:4 +dcid:NorthernAfrica,2023,16,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:55:4 +dcid:NorthernAfricaAndWesternAsia,2015,13.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:56:4 +dcid:NorthernAfricaAndWesternAsia,2016,13.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:57:4 +dcid:NorthernAfricaAndWesternAsia,2017,13.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:58:4 +dcid:NorthernAfricaAndWesternAsia,2018,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:59:4 +dcid:NorthernAfricaAndWesternAsia,2019,14.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:60:4 +dcid:NorthernAfricaAndWesternAsia,2020,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:61:4 +dcid:NorthernAfricaAndWesternAsia,2021,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:62:4 +dcid:NorthernAfricaAndWesternAsia,2022,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:63:4 +dcid:NorthernAfricaAndWesternAsia,2023,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:64:4 +dcid:SubSaharanAfrica,2015,22.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:65:4 +dcid:SubSaharanAfrica,2016,22,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:66:4 +dcid:SubSaharanAfrica,2017,21.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:67:4 +dcid:SubSaharanAfrica,2018,22.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:68:4 +dcid:SubSaharanAfrica,2019,22.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:69:4 +dcid:SubSaharanAfrica,2020,23,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:70:4 +dcid:SubSaharanAfrica,2021,23,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:71:4 +dcid:SubSaharanAfrica,2022,22.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:72:4 +dcid:SubSaharanAfrica,2023,23,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:73:4 +dcid:WesternAfrica,2015,26,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:74:4 +dcid:WesternAfrica,2016,25.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:75:4 +dcid:WesternAfrica,2017,25.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:76:4 +dcid:WesternAfrica,2018,26.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:77:4 +dcid:WesternAfrica,2019,26.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:78:4 +dcid:WesternAfrica,2020,26.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:79:4 +dcid:WesternAfrica,2021,26.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:80:4 +dcid:WesternAfrica,2022,26.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:81:4 +dcid:WesternAfrica,2023,26.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:82:4 +dcid:EasternAfrica,2015,18.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:83:4 +dcid:EasternAfrica,2016,18.4,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:84:4 +dcid:EasternAfrica,2017,18.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:85:4 +dcid:EasternAfrica,2018,18.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:86:4 +dcid:EasternAfrica,2019,18.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:87:4 +dcid:EasternAfrica,2020,18.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:88:4 +dcid:EasternAfrica,2021,18.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:89:4 +dcid:EasternAfrica,2022,18.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:90:4 +dcid:EasternAfrica,2023,18.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:91:4 +dcid:SouthernAfrica,2015,22.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:92:4 +dcid:SouthernAfrica,2016,21.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:93:4 +dcid:SouthernAfrica,2017,21.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:94:4 +dcid:SouthernAfrica,2018,22.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:95:4 +dcid:SouthernAfrica,2019,22.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:96:4 +dcid:SouthernAfrica,2020,22.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:97:4 +dcid:SouthernAfrica,2021,22.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:98:4 +dcid:SouthernAfrica,2022,22.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:99:4 +dcid:SouthernAfrica,2023,22.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:100:4 +dcid:MiddleAfrica,2015,19.5,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:101:4 +dcid:MiddleAfrica,2016,19.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:102:4 +dcid:MiddleAfrica,2017,19,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:103:4 +dcid:MiddleAfrica,2018,20.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:104:4 +dcid:MiddleAfrica,2019,21,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:105:4 +dcid:MiddleAfrica,2020,21.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:106:4 +dcid:MiddleAfrica,2021,21.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:107:4 +dcid:MiddleAfrica,2022,21.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:108:4 +dcid:MiddleAfrica,2023,21.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:109:4 +dcid:Asia,2015,14.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:110:4 +dcid:Asia,2016,13.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:111:4 +dcid:Asia,2017,13.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:112:4 +dcid:Asia,2018,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:113:4 +dcid:Asia,2019,14.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:114:4 +dcid:Asia,2020,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:115:4 +dcid:Asia,2021,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:116:4 +dcid:Asia,2022,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:117:4 +dcid:Asia,2023,14.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:118:4 +G00100100,2015,15,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:119:4 +G00100100,2016,14.5,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:120:4 +G00100100,2017,14.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:121:4 +G00100100,2018,14.4,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:122:4 +G00100100,2019,14.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:123:4 +G00100100,2020,14.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:124:4 +G00100100,2021,14.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:125:4 +G00100100,2022,14.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:126:4 +G00100100,2023,14.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:127:4 +dcid:CentralAsia,2015,9.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:128:4 +dcid:CentralAsia,2016,9.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:129:4 +dcid:CentralAsia,2017,9.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:130:4 +dcid:CentralAsia,2018,10,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:131:4 +dcid:CentralAsia,2019,10.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:132:4 +dcid:CentralAsia,2020,10.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:133:4 +dcid:CentralAsia,2021,10.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:134:4 +dcid:CentralAsia,2022,10.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:135:4 +dcid:CentralAsia,2023,10.3,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:136:4 +dcid:SouthernAsia,2015,15.5,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:137:4 +dcid:SouthernAsia,2016,14.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:138:4 +dcid:SouthernAsia,2017,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:139:4 +dcid:SouthernAsia,2018,14.8,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:140:4 +dcid:SouthernAsia,2019,14.7,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:141:4 +dcid:SouthernAsia,2020,14.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:142:4 +dcid:SouthernAsia,2021,14.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:143:4 +dcid:SouthernAsia,2022,14.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:144:4 +dcid:SouthernAsia,2023,14.6,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:145:4 +dcid:WesternAsia,2015,12.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:146:4 +dcid:WesternAsia,2016,11.5,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:147:4 +dcid:WesternAsia,2017,11.2,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:148:4 +dcid:WesternAsia,2018,13.1,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:149:4 +dcid:WesternAsia,2019,13.9,dcid:undata/AG_FLS_PCT.PRODUCT--_T,UN_UNIT_MEASURE_PT,,,UN_CENSOR_VALUE_TYPE-_Z,,,,UN_-N,UN_OBSERVATION_STATUS-NORMAL,,,SDG_q1-2026_OBS_AG_FLS_PCT.csv:150:4 diff --git a/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf b/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf new file mode 100644 index 0000000000..cc0cd205e7 --- /dev/null +++ b/scripts/un/un_dataset_validator/test_data/SDG_q1-2026_OBS_AG_FLS_PCT_data_stat_vars.mcf @@ -0,0 +1,35 @@ +Node: dcid:undata/AG_FLS_PCT.PRODUCT--AGG_ANIMAL_PROD +typeOf: dcid:StatisticalVariable +populationType: dcid:UN_SDG_-AG_FLS_PCT +measuredProperty: dcid:value +statType: dcid:measuredValue +product: dcid:UN_PRODUCT-AGG_ANIMAL_PROD + +Node: dcid:undata/AG_FLS_PCT.PRODUCT--AGG_CRL_PUL +typeOf: dcid:StatisticalVariable +populationType: dcid:UN_SDG_-AG_FLS_PCT +measuredProperty: dcid:value +statType: dcid:measuredValue +product: dcid:UN_PRODUCT-AGG_CRL_PUL + +Node: dcid:undata/AG_FLS_PCT.PRODUCT--AGG_FRT_VGT +typeOf: dcid:StatisticalVariable +populationType: dcid:UN_SDG_-AG_FLS_PCT +measuredProperty: dcid:value +statType: dcid:measuredValue +product: dcid:UN_PRODUCT-AGG_FRT_VGT + +Node: dcid:undata/AG_FLS_PCT.PRODUCT--AGG_RT_TBR +typeOf: dcid:StatisticalVariable +populationType: dcid:UN_SDG_-AG_FLS_PCT +measuredProperty: dcid:value +statType: dcid:measuredValue +product: dcid:UN_PRODUCT-AGG_RT_TBR + +Node: dcid:undata/AG_FLS_PCT.PRODUCT--_T +typeOf: dcid:StatisticalVariable +populationType: dcid:UN_SDG_-AG_FLS_PCT +measuredProperty: dcid:value +statType: dcid:measuredValue +product: dcid:UN_PRODUCT-_T + diff --git a/scripts/un/un_dataset_validator/validation_implementation_analysis.md b/scripts/un/un_dataset_validator/validation_implementation_analysis.md new file mode 100644 index 0000000000..4055d649e5 --- /dev/null +++ b/scripts/un/un_dataset_validator/validation_implementation_analysis.md @@ -0,0 +1,98 @@ +# Custom Data Commons (DC) Validation Implementation Analysis + +## 1. Executive Summary +This document outlines a detailed and comprehensive strategy for implementing an automated validation script in Python for the Custom Data Commons datasets. The script verifies that the data transformation pipeline correctly converts raw UN dataset inputs into the required Data Commons formats (MCF and CSV). + +*Note: As instructed, all checklist items marked with "Ask Ajai" (Checks 1, 4.1, 6.1, 8, 10.1, and 13) have been explicitly excluded from this analysis.* + +--- + +## 2. Validation Flow and File Definitions + +The validation process involves comparing four distinct categories of files. Let's trace the exact flow using the `SDG_q1-2026` dataset (specifically the `AG_FLS_INDEX` series) as an example. + +### A. The Input Files (The "Source of Truth") +The raw input dataset for the UN series contains the initial data points. While the raw files might be staged in buckets or upstream folders, we know their structure based on the transcript and the traceback from the generated files. +* **Example Input Source:** `SDG_q1-2026_OBS_AG_FLS_INDEX.csv` (referenced by the `#input` column in the output CSV). +* **Key Columns Expected:** `series`, `geography` (e.g., `D0`), `timePeriod`, `obsValue`, and various dimensions (like `product`) and attributes (like `censoredValueType`). + +### B. The Structure Definitions (DSD & Codelists) +The Data Structure Definition (DSD) dictates how each column in the input file should be treated. +* **Flow:** The validation script must read the DSD to classify columns into two buckets: `ROLE="dimension"` (e.g., `series`, `geography`, `timePeriod`, `product`) and `ROLE="Attribute"` (e.g., `censoredValueType`, `unitMultiplier`). + +### C. The Property-Value (PV) Maps +These maps provide the exact translation from UN source codes to Data Commons DCIDs. +* **Example File:** `/all_data/pvmap/un_geography_pvmap.csv`. +* **Flow:** When validating geography, the script cross-references the input `geography` value against this PV map to determine the expected output DCID. + +### D. The Output Files (The Data to Validate) +The generated files live in the `processed_data/` directory. +1. **Output CSV (`SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`):** Contains the transcoded data. + * **Columns generated:** `observationAbout`, `observationDate`, `value`, `variableMeasured` (StatVar DCID), and extra attribute columns like `censoredValueType`. +2. **Output MCF (`SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf`):** Contains the definitions for the Statistical Variables (StatVars). + * **Example Node:** `Node: dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD` + +--- + +## 3. Implementation Strategy & Flow by Checklist Rule + +### Check 2: `SERIES` mapped to `populationType` +* **Flow:** + 1. Extract the series code from the input (e.g., `AG_FLS_INDEX`). + 2. Locate the corresponding Statistical Variable in the output MCF. + 3. Validate that the `populationType` property exists and is correctly prefixed. +* **SDG Example:** In `SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf`, the node `dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD` must have the property `populationType: dcid:UN_SDG_SERIES-AG_FLS_INDEX`. +* **Validation Logic:** Assert `output_mcf_node['populationType'] == f"dcid:UN_SDG_SERIES-{input_series}"`. + +### Check 3: `GEOGRAPHY` mapped to `observationAbout` +* **Flow:** + 1. Read the `geography` column from the input file. + 2. Look up the corresponding Data Commons DCID using `un_geography_pvmap.csv`. + 3. Verify that the expected DCID populates the `observationAbout` column in the output CSV. +* **SDG Example:** If the input geography is `D0` (World), the PV map resolves this to `Earth`. In `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`, the `observationAbout` column must equal `dcid:Earth`. +* **Validation Logic:** If a regional/national geography code fails to resolve via the PV map, the script must flag it and record the unresolved geography code to a separate error log. + +### Check 4: `TIME_PERIOD` mapped to `observationDate` +* **Flow:** Compare the time string from the input to the output. +* **SDG Example:** If the input `timePeriod` is `2015`, the output CSV (`SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`) must have `observationDate` = `2015`. +* **Validation Logic:** Assert `input_row['timePeriod'] == output_row['observationDate']`. (Complex un-mappable date formats like `2024-25/P3M` are ignored per "Ask Ajai" rules). + +### Check 5: `OBS_VALUE` mapped to `value` +* **Flow:** Ensure numerical fidelity between input and output. +* **SDG Example:** In `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`, a row contains `value: 100`. This must exactly match the `OBS_VALUE` from the corresponding row in the input CSV. +* **Validation Logic:** Use `pandas.to_numeric()` to ensure the generated output `value` is an exact float/int equivalent of the input. + +### Check 6: Dimensions as Constraint Properties +* **Flow:** + 1. The DSD marks specific columns (other than time, geo, value, and series) as dimensions. + 2. These dimensions must appear as properties attached to the StatVar Node in the MCF. +* **SDG Example:** The DSD for `AG_FLS_INDEX` defines `product` as a dimension. In the MCF (`SDG_q1-2026_OBS_AG_FLS_INDEX_data_stat_vars.mcf`), the script must find the property attached to the StatVar: `product: dcid:UN_PRODUCT-AGG_ANIMAL_PROD`. +* **Validation Logic:** Assert that the MCF node contains `product` and that its value adheres to the `_-` format (e.g., `UN_PRODUCT-...`). + +### Check 7: `UNIT_MULTIPLIER` Application +* **Flow:** If the input data has a multiplier (e.g., Thousands), the output `value` must be pre-calculated. +* **Validation Logic:** + 1. Look up the input's `UNIT_MULTIPLIER` code in `CL_MULT_pvmap_multiply.csv`. + 2. Calculate: `Expected_Value = float(input['OBS_VALUE']) * integer_multiplier`. + 3. Assert `Expected_Value == float(output['value'])`. + +### Check 9: `FREQUENCY` to `observationPeriod` +* **Flow:** Ensure the input frequency maps to the correct DC `observationPeriod`. +* **Validation Logic:** Check the PV map (`CL_FREQUENCY_pvmap_obsperiod.csv`). +* **Important Anomaly Handling:** The Python script must dynamically check the output CSV headers for known typos like `opservationPeriod` (which exists in the current output files). It should fail the validation if the correct `observationPeriod` header is missing, flagging the typo. + +### Check 10: Attributes as Output Columns +* **Flow:** + 1. The DSD dictates which columns are "Attributes" (e.g., not attached to the MCF StatVar, but carried over to the CSV). + 2. The script checks the output CSV headers for these columns. +* **SDG Example:** `censoredValueType` is an attribute. In `SDG_q1-2026_OBS_AG_FLS_INDEX_data.csv`, there is an explicitly generated column named `censoredValueType` containing values like `UN_CENSORED_VALUE_TYPE-_Z`. +* **Validation Logic:** `assert 'censoredValueType' in output_csv.columns`. + +### Check 11: StatVar DCID Format & Special Characters +* **Flow:** The DCID generated for each variable must follow the strict `//[.--__…]` template without illegal characters. +* **SDG Example:** The script reads `dcid:undata/sdg/AG_FLS_INDEX.PRODUCT--AGG_ANIMAL_PROD` from the MCF. +* **Validation Logic:** Regex validation. Ensure that original special characters (e.g., in the product code) were accurately replaced by underscores (`_`). + +### Check 12: Name Property Matches DSD/CL +* **Flow:** The generated `name` in the MCF should be human-readable, mapped directly from the Codelist descriptions. +* **Validation Logic:** Cross-reference the dimension codes (e.g., `AGG_ANIMAL_PROD`) against the UN Codelist descriptions. Ensure that these descriptions are concatenated correctly into the `name` property of the StatVar MCF node. (Note: The script should log a warning rather than failing if the name is entirely missing, as this is a known pipeline issue). \ No newline at end of file