From dc35b260dc90c3853944a04d88b85c17d5dbe0f1 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 28 Jul 2026 09:46:33 +0000 Subject: [PATCH 1/6] fixed bug --- tools/import_validation/validator_goldens.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/import_validation/validator_goldens.py b/tools/import_validation/validator_goldens.py index 766e9f890d..281cb92f7d 100644 --- a/tools/import_validation/validator_goldens.py +++ b/tools/import_validation/validator_goldens.py @@ -498,6 +498,19 @@ def validate_goldens(inputs: str | dict, golden_files_list = file_util.file_get_matching(golden_files) golden_nodes = load_nodes_from_file(golden_files_list) + # Normalize list-like string values (e.g. [A, B]) so sequence doesn't matter. + for nodes in [input_nodes, golden_nodes]: + for node in nodes.values(): + for k, v in node.items(): + if isinstance(v, str) and v.startswith('[') and v.endswith(']'): + if ',' in v or v == '[]': + content = v[1:-1].strip() + if content: + parts = sorted([p.strip() for p in content.split(',')]) + node[k] = '[' + ', '.join(parts) + ']' + else: + node[k] = '[]' + # Run the core comparison logic. missing_goldens = validator_compare_nodes(input_nodes, golden_nodes, config, counters) From cad7776d40dbe511631f5f62f3b356dc7853c917 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Tue, 28 Jul 2026 10:02:56 +0000 Subject: [PATCH 2/6] fixed bug --- tools/import_validation/validator_goldens.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/import_validation/validator_goldens.py b/tools/import_validation/validator_goldens.py index 281cb92f7d..d8c1f236cd 100644 --- a/tools/import_validation/validator_goldens.py +++ b/tools/import_validation/validator_goldens.py @@ -503,13 +503,12 @@ def validate_goldens(inputs: str | dict, for node in nodes.values(): for k, v in node.items(): if isinstance(v, str) and v.startswith('[') and v.endswith(']'): - if ',' in v or v == '[]': - content = v[1:-1].strip() - if content: - parts = sorted([p.strip() for p in content.split(',')]) - node[k] = '[' + ', '.join(parts) + ']' - else: - node[k] = '[]' + content = v[1:-1].strip() + if content: + parts = sorted([p.strip() for p in content.split(',')]) + node[k] = '[' + ', '.join(parts) + ']' + else: + node[k] = '[]' # Run the core comparison logic. missing_goldens = validator_compare_nodes(input_nodes, golden_nodes, config, From 5c544e641c4ad3bd1f181a277953e6587a45dd56 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 5 Aug 2026 06:01:02 +0000 Subject: [PATCH 3/6] added runner.py --- tools/import_validation/runner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/import_validation/runner.py b/tools/import_validation/runner.py index bcf83fb06e..c2f5ba271a 100644 --- a/tools/import_validation/runner.py +++ b/tools/import_validation/runner.py @@ -191,7 +191,8 @@ def _load_differ_df_from_mcf(self, input_dir: str) -> pd.DataFrame: stats[current_var][diff_type] += 1 if not stats: - return pd.DataFrame() + return pd.DataFrame( + columns=['StatVar', 'ADDED', 'DELETED', 'MODIFIED']) rows = [] for var, counts in stats.items(): From b000070ba7781762c0ab4c6e7116234daffba69b Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 5 Aug 2026 13:47:22 +0000 Subject: [PATCH 4/6] import_validation: simplify list normalization in goldens validation --- tools/import_validation/validator_goldens.py | 12 ------------ tools/statvar_importer/mcf_diff.py | 11 ++++++++++- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/tools/import_validation/validator_goldens.py b/tools/import_validation/validator_goldens.py index d8c1f236cd..766e9f890d 100644 --- a/tools/import_validation/validator_goldens.py +++ b/tools/import_validation/validator_goldens.py @@ -498,18 +498,6 @@ def validate_goldens(inputs: str | dict, golden_files_list = file_util.file_get_matching(golden_files) golden_nodes = load_nodes_from_file(golden_files_list) - # Normalize list-like string values (e.g. [A, B]) so sequence doesn't matter. - for nodes in [input_nodes, golden_nodes]: - for node in nodes.values(): - for k, v in node.items(): - if isinstance(v, str) and v.startswith('[') and v.endswith(']'): - content = v[1:-1].strip() - if content: - parts = sorted([p.strip() for p in content.split(',')]) - node[k] = '[' + ', '.join(parts) + ']' - else: - node[k] = '[]' - # Run the core comparison logic. missing_goldens = validator_compare_nodes(input_nodes, golden_nodes, config, counters) diff --git a/tools/statvar_importer/mcf_diff.py b/tools/statvar_importer/mcf_diff.py index e7eaa825f8..39029df94f 100644 --- a/tools/statvar_importer/mcf_diff.py +++ b/tools/statvar_importer/mcf_diff.py @@ -299,7 +299,16 @@ def fingerprint_node(pvs: dict, for p in sorted(normalized_pvs.keys()): if p not in ignore_props: if not compare_props or p in compare_props: - fp.append(f'{p}={normalized_pvs[p]}') + val = normalized_pvs[p] + # Normalize list-like string values (e.g. [A, B]) so sequence doesn't matter. + if isinstance(val, str) and val.startswith('[') and val.endswith(']'): + content = val[1:-1].strip() + if content: + parts = sorted([p.strip() for p in content.split(',')]) + val = '[' + ', '.join(parts) + ']' + else: + val = '[]' + fp.append(f'{p}={val}') return ';'.join(fp) From b9818167c1c928a3820a2465bc80eec0d8e2b63e Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 5 Aug 2026 15:32:29 +0000 Subject: [PATCH 5/6] statvar_importer: fix list sorting and bracketed list normalization in mcf_file_util --- tools/statvar_importer/mcf_file_util.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/statvar_importer/mcf_file_util.py b/tools/statvar_importer/mcf_file_util.py index 7f6177c52c..8120b2c055 100644 --- a/tools/statvar_importer/mcf_file_util.py +++ b/tools/statvar_importer/mcf_file_util.py @@ -826,7 +826,7 @@ def normalize_list(value: str, sort: bool = True) -> str: value_list = get_value_list(value) has_quotes = True else: - value_list = value.split(',') + value_list = [v.strip() for v in value.split(',')] values = [] if sort: value_list = sorted(value_list) @@ -931,6 +931,10 @@ def normalize_value( if value[0] == '"' and value[-1] == '"' and len(value) > 100: # Retain very long strings, such as geoJsonCoordinates, as is. return value + if value.startswith('[') and value.endswith(']') and ',' in value: + inner_list = value[1:-1].strip() + normalized_list = normalize_list(inner_list) + return f'[{normalized_list}]' if ',' in value and maybe_list: return normalize_list(value) if value[0] == '[': From 26036e414629ceaf11a61cc59f2f0ef5689a4743 Mon Sep 17 00:00:00 2001 From: Nivedita Singh Date: Wed, 5 Aug 2026 15:40:36 +0000 Subject: [PATCH 6/6] statvar_importer: revert list normalization changes in mcf_diff.py --- tools/statvar_importer/mcf_diff.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/tools/statvar_importer/mcf_diff.py b/tools/statvar_importer/mcf_diff.py index 39029df94f..e7eaa825f8 100644 --- a/tools/statvar_importer/mcf_diff.py +++ b/tools/statvar_importer/mcf_diff.py @@ -299,16 +299,7 @@ def fingerprint_node(pvs: dict, for p in sorted(normalized_pvs.keys()): if p not in ignore_props: if not compare_props or p in compare_props: - val = normalized_pvs[p] - # Normalize list-like string values (e.g. [A, B]) so sequence doesn't matter. - if isinstance(val, str) and val.startswith('[') and val.endswith(']'): - content = val[1:-1].strip() - if content: - parts = sorted([p.strip() for p in content.split(',')]) - val = '[' + ', '.join(parts) + ']' - else: - val = '[]' - fp.append(f'{p}={val}') + fp.append(f'{p}={normalized_pvs[p]}') return ';'.join(fp)