diff --git a/import-automation/executor/requirements.txt b/import-automation/executor/requirements.txt index 7ee91417a2..9868d3656c 100644 --- a/import-automation/executor/requirements.txt +++ b/import-automation/executor/requirements.txt @@ -1,6 +1,7 @@ # Requirements for Python scripts in this repo that have automation enabled! absl-py +anyascii arcgis2geojson beautifulsoup4 chardet diff --git a/scripts/un/codes/codelist_pvmap_template.py b/scripts/un/codes/codelist_pvmap_template.py new file mode 100644 index 0000000000..3a9b0f4196 --- /dev/null +++ b/scripts/un/codes/codelist_pvmap_template.py @@ -0,0 +1,23 @@ +# Template for converting UN codelist files to PVMap for statvar processor. +# The pvmap will have columns to generate statvar constraint propoerty:values +# and names. +# The pvmap will also have columns to generate schema MCF with a tMCF. +{ + 'key': '{CONCEPT}:{CODE}', + 'UnConceptProp': 'Property', + 'UnConcept': '"{CONCEPT}"', + 'UnCodeProp': 'UnCode', + 'UnCode': '"{CODE}"', + 'ConstraintProp': '{PROPERTY}', + 'ConstraintPropValue': 'to_dcid(NAMESPACE+"_"+CONCEPT+"-"+CODE)', + 'ConstraintPropType': 'TypeOf', + 'ConstraintPropEnum': 'str(ConstraintProp[0].upper() + ConstraintProp[1:]+"Enum")', + 'NameProp': 'ValueName_{CONCEPT}', + 'ConstraintValueName': '"{NAME_EN}"', + 'DescriptionProp': 'Desc_{CONCEPT}', + 'ConstraintValueDescription': 'quote(anyascii(DESCRIPTION))', + # Eond of line prop:value when description is empty. + '#End': 'End', + 'Dummy': '.', +} + diff --git a/scripts/un/codes/codelist_schema.tmcf b/scripts/un/codes/codelist_schema.tmcf new file mode 100644 index 0000000000..038dd3a5d9 --- /dev/null +++ b/scripts/un/codes/codelist_schema.tmcf @@ -0,0 +1,17 @@ +Node: E:UN->E0 +typeOf: C:UN->ConstraintPropEnum +dcid: C:UN->ConstraintPropValue +unCode: C:UN->UnCode +name: C:UN->ConstraintValueName +description: C:UN->ConstraintValueDescription + +Node: E:UN->E1 +typeOf: C:UN->UnConceptProp +dcid: C:UN->ConstraintProp +unConcept: C:UN->UnConcept +rangeIncludes: C:UN->ConstraintPropEnum + +Node: E:UN->E2 +typeOf: schema:Class +dcid: C:UN->ConstraintPropEnum +subClassOf: schema:Enumeration diff --git a/scripts/un/codes/dsd_property_pvmap.py b/scripts/un/codes/dsd_property_pvmap.py new file mode 100644 index 0000000000..aa1b9ef914 --- /dev/null +++ b/scripts/un/codes/dsd_property_pvmap.py @@ -0,0 +1,22 @@ +# Template for converting UN DSD file with column metadata +# to PVMap for statvar processor. +# The pvmap will have columns to generate statvar constraint +# propoerties with names. +# The pvmap will also have columns to generate schema MCF with a tMCF. +{ + 'key': '{CONCEPT}', + 'UnCodeProp': 'UnConceptCode', + 'UnCode': '"{CONCEPT}"', + 'ConceptProp': 'UnConceptProperty', + 'ConstraintProp': '{PROPERTY}', + 'ConstraintPropType': 'Property', + 'ConstraintPropEnum': 'str(ConstraintProp[0].upper() + ConstraintProp[1:]+"Enum")', + 'ConceptNameProp': 'PropertyName_{CONCEPT}', + 'ConceptName': '"{NAME_EN}"', + 'DescriptionProp': 'PropertyDesc_{CONCEPT}', + 'ConceptDescription': '{DESCRIPTION}', + # Eond of line prop:value when description is empty. + '#End': 'End', + 'Dummy': '.', +} + diff --git a/scripts/un/codes/dsd_property_pvmap_template.py b/scripts/un/codes/dsd_property_pvmap_template.py new file mode 100644 index 0000000000..708beac19f --- /dev/null +++ b/scripts/un/codes/dsd_property_pvmap_template.py @@ -0,0 +1,39 @@ +# Template for converting UN DSD file with column metadata +# to PVMap for statvar processor. +# The pvmap will have columns to generate statvar constraint +# propoerties with names. +# The pvmap will also have columns to generate schema MCF with a tMCF. +{ + 'key': + '{CONCEPT}', + 'UnCodeProp': + 'UnConceptCode', + 'UnCode': + '"{CONCEPT}"', + 'ConceptProp': + 'UnConceptProperty', + 'ConstraintProp': + '{PROPERTY}', + 'ConstraintPropType': + 'Property', + 'ConstraintPropEnum': + 'str(ConstraintProp[0].upper() + ConstraintProp[1:]+"Enum")', + 'ConceptNameProp': + 'PropertyName_{CONCEPT}', + 'ConceptName': + '"{NAME_EN}"', + 'DescriptionProp': + 'ValueDesc_{CONCEPT}', + 'ConceptDescription': + 'quote(anyascii(DESCRIPTION))', + # Initialize ValueName for specific codes for a concept to empty string + 'CodeNameProp': + 'ValueName_{CONCEPT}', + 'DefaultName': + '""', + # End of line prop:value when description is empty. + '#End': + 'End', + 'Dummy': + '.', +} diff --git a/scripts/un/codes/dsd_property_schema.tmcf b/scripts/un/codes/dsd_property_schema.tmcf new file mode 100644 index 0000000000..8491b4b64f --- /dev/null +++ b/scripts/un/codes/dsd_property_schema.tmcf @@ -0,0 +1,15 @@ +Node: E:DSD->E0 +dcid: C:DSD->ConstraintProp +typeOf: C:DSD->ConstraintPropType +name: C:DSD->ConstraintProp +domainIncludes: dcid:UNSeries +alternateName: C:DSD->ConceptName +description: C:DSD->ConceptDescription +rangeIncludes: C:DSD->ConstraintPropEnum + +Node: E:DSD->E1 +dcid: C:DSD->ConstraintPropEnum +typeOf: schema:Class +subClassOf: schema:Enumeration +description: C:DSD->ConceptDescription + diff --git a/scripts/un/codes/generate_codelist_map.py b/scripts/un/codes/generate_codelist_map.py new file mode 100644 index 0000000000..88cb3b6bfc --- /dev/null +++ b/scripts/un/codes/generate_codelist_map.py @@ -0,0 +1,229 @@ +"""Script to generate codelist mapings for a specific codelist file.""" + +import os +import re +import sys +import unicodedata + +from absl import app +from absl import flags +from absl import logging +from anyascii import anyascii +from pprint import pprint + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) +sys.path.append(os.path.dirname(_SCRIPT_DIR)) +sys.path.append(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +_DATA_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +sys.path.append(_DATA_DIR) +sys.path.append(os.path.join(_DATA_DIR, 'util')) +sys.path.append(os.path.join(_DATA_DIR, 'tools', 'statvar_importer')) + +import file_util +import mcf_file_util +import eval_functions + +from counters import Counters + +flags.DEFINE_string('input_codelist', '', 'CSV file with codelist.') +flags.DEFINE_string('output_pvmap', '', 'Output pvmap csv.') +flags.DEFINE_string('namespace', 'un', 'Namespace prefix for agency') +flags.DEFINE_string('pvmap_template', '', 'Python file with pvmap template.') +flags.DEFINE_integer('logging_level', logging.INFO, 'Logging level.') + +_FLAGS = flags.FLAGS + +_DEFAULT_CODE_PROPS = [ + 'CONCEPT', + 'CODE', + 'NAME_EN', + 'PARENT', + 'SORT_ORDER', + 'NAME_FR', + 'NAME_ES', + 'DESCRIPTION', +] + +# Map from a code to a pvmap +_DEFAULT_CODE_PVMAP = { + 'key': + '{CONCEPT}:{CODE}', + 'UnConceptProp': + 'Property', + 'UnConcept': + '"{CONCEPT}"', + 'UnCodeProp': + 'UnCode', + 'UnCode': + '"{CODE}"', + 'ConstraintProp': + '{PROPERTY}', + 'ConstraintPropValue': + 'to_dcid(NAMESPACE+"_"+CONCEPT+"-"+CODE)', + 'ConstraintPropType': + 'TypeOf', + 'ConstraintPropEnum': + 'str(ConstraintProp[0].upper() + ConstraintProp[1:]+"Enum")', + 'NameProp': + 'ValueName_{CONCEPT}', + 'ConstraintValueName': + '"{NAME_EN}"', + 'DescriptionProp': + 'ValueDesc_{CONCEPT}', + 'ConstraintValueDescription': + '{DESCRIPTION}', + 'End': + 'End', + 'Dummy': + '.', +} + +# Mapping from concept to properties. +# If not set it map, the concept is used as the property. +_DEFAULT_CONCEPT_PROP_MAP = { + 'SERIES': 'populationType', + 'UNIT_MEASURE': 'unit', +} + +def quote(value: str) -> str: + """Returns a string in double quotes.""" + value = value.strip().strip('"').strip() + return f'"{value}"' + +def to_property(concept: str) -> str: + """Returns a property for the concept.""" + c = eval_functions.str_to_camel_case(concept.lower().replace('_', ' ')) + return c[0].lower() + c[1:] + + +def to_dcid(code: str) -> str: + """Replace any non alphanumeric characters with '_'""" + value = re.sub(r'[^A-Za-z0-9\._:-]+', '_', code) + return value[0].upper() + value[1:] + + +def clean_value_str(val: str, + regex: str = 'r[^A-Za-z0-9()[]".-]+', + replace: str = '_') -> str: + """Cleanup value string to remove redundant characters.""" + val = val.strip() + if val[0] == '"' and val[-1] == '"': + val = '"' + val[1:-1].strip() + '"' + val = re.sub(regex, replace, val) + return val + + +_EVAL_FUNCTIONS = dict(eval_functions.EVAL_GLOBALS) +_EVAL_FUNCTIONS.update({ + 'to_property': to_property, + 'to_dcid': to_dcid, + 'clean_value_str': clean_value_str, + 'quote': quote, + + # Additional modules for text manipulations + 'unicodedata': unicodedata, + 'anyascii': anyascii, +}) + + +def get_value(tpl_val: str, input_pvs: dict) -> str: + """Retuns a value with the pvs applied.""" + value = tpl_val + if '{' in tpl_val: + # Format string + try: + value = tpl_val.format(**input_pvs) + except Exception as e: + logging.error( + f'Failed to format "{tpl_val}" using dict: {input_pvs}, error:{e}' + ) + value = '' + elif '(' in tpl_val: + # Evaluate a function + try: + prop, value = eval_functions.evaluate_statement( + tpl_val, input_pvs, _EVAL_FUNCTIONS) + except Exception as e: + lgging.error( + f'Failed to evaluate "{tpl_val}" using dict: {pvs}, error:{e}') + value = '' + if value: + # Cleanup value + value = clean_value_str(value) + return value + + +def generate_code_map(code_pvs: dict, + namespace: str = 'un', + template: dict = _DEFAULT_CODE_PVMAP) -> dict: + """Returns a pvmap pvs for a single code. + A code has keys listed in _DEFAULT_CODE_PROPS + It returns a dictionary with the keys in template. + """ + output_pvs = dict() + input_pvs = dict(code_pvs) + input_pvs.setdefault('namespace', namespace.lower()) + input_pvs.setdefault('NAMESPACE', namespace.upper()) + concept = code_pvs.get('CONCEPT') + concept_prop = _DEFAULT_CONCEPT_PROP_MAP.get(concept) + if not concept_prop: + concept_prop = to_property(concept) + input_pvs['PROPERTY'] = concept_prop + for tpl_prop, tpl_val in template.items(): + tpl_prop = get_value(tpl_prop, input_pvs) + value = get_value(tpl_val, input_pvs) + output_pvs[tpl_prop] = value + input_pvs[tpl_prop] = value + logging.log(2, f'Mapped {tpl_prop} using {tpl_val} to {value}') + return output_pvs + + +def generate_codelist_pvmap(cl_file: str, + output: str, + namespace: str = 'un', + template_file: str = None) -> dict: + """Generate a pvmap file for a codelist.""" + counters = Counters() + + input_codes = file_util.file_load_csv_dict(cl_file, key_index=True) + logging.info(f'Loaded {len(input_codes)} from codelist: {cl_file}') + counters.add_counter('input-codes', len(input_codes)) + + pvmap_template = _DEFAULT_CODE_PVMAP + if template_file: + pvmap_template = file_util.file_load_py_dict(template_file) + + logging.info(f'Using template: {pprint(pvmap_template)}') + + output_pvs = {} + for index, code_pvs in input_codes.items(): + pvs = generate_code_map(code_pvs, namespace, pvmap_template) + output_pvs[index] = pvs + logging.debug(f'Mapped {code_pvs} to {pvs}') + + # Write to output file + if output: + file_util.file_write_csv_dict(output_pvs, output) + + # Get unique counts across output columns + unique_counts = dict() + for index, pvs in output_pvs.items(): + for prop, val in pvs.items(): + if val: + unique_counts.setdefault(prop, set()).add(val) + for prop, vals in unique_counts.items(): + counters.add_counter(f'output-unique-{prop}', len(vals)) + + counters.add_counter('output-rows', len(output_pvs)) + counters.print_counters() + + +def main(_): + logging.set_verbosity(_FLAGS.logging_level) + generate_codelist_pvmap(_FLAGS.input_codelist, _FLAGS.output_pvmap, + _FLAGS.namespace, _FLAGS.pvmap_template) + + +if __name__ == '__main__': + app.run(main) diff --git a/scripts/un/codes/generate_statvar_groups.py b/scripts/un/codes/generate_statvar_groups.py new file mode 100644 index 0000000000..4675baa196 --- /dev/null +++ b/scripts/un/codes/generate_statvar_groups.py @@ -0,0 +1,317 @@ +"""Script to generate statvar groups for UN statvars.""" + +import itertools +import os +import re +import sys + +from absl import app +from absl import flags +from absl import logging + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) +sys.path.append(os.path.dirname(_SCRIPT_DIR)) +sys.path.append(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +_DATA_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +sys.path.append(_DATA_DIR) +sys.path.append(os.path.join(_DATA_DIR, 'util')) +sys.path.append(os.path.join(_DATA_DIR, 'tools', 'statvar_importer')) + +import file_util +from mcf_file_util import add_namespace, strip_namespace, get_node_dcid +from mcf_file_util import load_mcf_nodes, write_mcf_nodes, add_mcf_node + +from config_map import ConfigMap +from counters import Counters + +flags.DEFINE_string('input_statvar_mcf', '', 'MCF files with statvar nodes.') +flags.DEFINE_string('output_statvar_group_mcf', '', + 'Output MCF files for statvar groups.') +flags.DEFINE_string('input_schema_mcf', '', + 'Schema file with names for properties.') +flags.DEFINE_string('statvar_root', 'dc/g/Root', 'Root for the statvar group.') +flags.DEFINE_string('statvar_group_prefix', 'custom/g/undata', + 'Prefix for the statvar group.') +flags.DEFINE_string('statvar_dcid_remove_prefix', '', + 'Prefix for the statvar group.') +flags.DEFINE_list('statvar_property_order', ['populationType'], + 'Statvar properties ordered by group heirarchy.') +flags.DEFINE_bool( + 'statvar_group_permutations', True, + 'Generate statvar groups for all permutations of properties.') +flags.DEFINE_bool( + 'statvar_add_linked_member_of', True, + 'Add linkedMemberOf property to statvars to all its parent groups.') +flags.DEFINE_integer('logging_level', logging.INFO, 'Logging level.') + +_FLAGS = flags.FLAGS +"""Wrapper to generate statvar groups for UN StatVars.""" + +_DEFAULT_IGNORE_PROP = { + 'Node': '', + 'dcid': '', + 'typeOf': '', + 'memberOf': '', + 'footnote': '', + 'description': '', + 'name': '', + 'measuredProperty': 'value', + 'statType': 'measuredValue', +} + + +def get_default_statvar_group_config() -> dict: + """Returns the default statvar group config.""" + return { + 'svg_root': _FLAGS.statvar_root, + 'svg_prefix': _FLAGS.statvar_group_prefix, + 'svg_properties': _FLAGS.statvar_property_order, + 'statvar_dcid_remove_prefix': _FLAGS.statvar_dcid_remove_prefix, + 'statvar_group_permutations': _FLAGS.statvar_group_permutations, + 'statvar_add_linked_member_of': _FLAGS.statvar_add_linked_member_of, + } + + +def to_snake_case(text: str, delim: str = '_', upper: bool = True) -> str: + """Returns a string in sentence case.""" + # convert camelCase + sentence = re.sub(r'(?<=[a-z0-9])(?=[A-Z])', delim, text) + + # convert '_' to spaces + sentence = re.sub(r'[_ ]+', delim, sentence) + sentence = sentence.strip() + if upper: + return sentence.upper() + return sentence + + +def to_quoted(text: str) -> str: + """Returns quoted string.""" + if not text: + return text + text = text.strip().strip('"').strip().replace('"', "'") + if text: + return '"' + text + '"' + return '' + + +class UNStatVarGroupGenerator: + + def __init__( + self, + config_dict: dict = {}, + counters: Counters = None, + ): + self._config = ConfigMap() + self._config.update_config(config_dict) + self._counters = counters + if counters is None: + self._counters = Counters() + # dictionary of schema nodes keyed by dcid. + self._schema_nodes = {} + # dictionary of statvar groups created. + self._statvar_groups = {} + + def load_schema_mcf(self, mcf: str) -> dict: + """Loads schema nodes from MCF files.""" + load_mcf_nodes(mcf, nodes=self._schema_nodes) + self._counters.add_counter('input-schema-nodes', + len(self._schema_nodes)) + return self._schema_nodes + + def get_schema_node(self, dcid: str) -> dict: + """Returns a schema node for the dcid.""" + if not dcid: + return None + node = self._schema_nodes.get(strip_namespace(dcid)) + if not node: + node = self._schema_nodes.get(add_namespace(dcid)) + return node + + def get_schema_name(self, dcid: str) -> str: + """Returns the name for the dcid fomr the schema.""" + node = self.get_schema_node(dcid) + if not node: + return '' + name = node.get('alternateName') + if not name: + name = node.get('name', '') + if not name: + # convert the dcid to a name string + remove_prefix = self._config.get('statvar_dcid_remove_prefix', '') + name = re.sub(remove_prefix, '', dcid[dcid.rfind('/') + 1:]) + name = to_snake_case(name).capitalize() + return name.strip('"').strip() + + def add_statvar_group(self, pvs: dict): + """Add a statvar group to schema.""" + add_mcf_node(pvs, self._schema_nodes) + add_mcf_node(pvs, self._statvar_groups) + + def get_statvar_groups(self) -> dict: + """Returns the new statvar groups created.""" + return self._statvar_groups + + def get_statvar_group_node(self, dcid, name, parent) -> dict: + return { + 'Node': add_namespace(dcid), + 'typeOf': 'dcid:StatVarGroup', + 'name': to_quoted(name), + 'specializationOf': add_namespace(parent), + } + + def generate_prop_value_svg(self, pvs: dict, grp_props: list, + svg_parent: str, svg_prefix: str) -> list[str]: + """Returns statvar group dcids for the property values in the list.""" + svg_grps = [] + strip_prefix = self._config.get('svg_dcid_remove_prefix', '') + depth = 0 + for prop in grp_props: + val = strip_namespace(pvs.get(prop, '')) + if not val: + continue + # Create svg for the property + prop_id = re.sub(strip_prefix, '', to_snake_case(prop)) + svg_dcid = svg_prefix + prop_id + svg_name = self.get_schema_name(prop) + self.add_statvar_group( + self.get_statvar_group_node(svg_dcid, svg_name, svg_parent)) + depth += 1 + self._counters.add_counter( + f'generated-statvar-groups-depth-{depth}', 1) + svg_parent = svg_dcid + svg_prefix = svg_dcid + self._config.get( + 'statvar_dcid_value_delimiter', '--') + + # Generate statvar group for value + val_id = re.sub(strip_prefix, '', val) + svg_dcid = svg_prefix + val_id + svg_name = self.get_schema_name(val) + self.add_statvar_group( + self.get_statvar_group_node(svg_dcid, svg_name, svg_parent)) + svg_grps.append(svg_dcid) + depth += 1 + self._counters.add_counter( + f'generated-statvar-groups-depth-{depth}', 1) + svg_parent = svg_dcid + svg_prefix = svg_dcid + self._config.get('statvar_dcid_delimiter', + '__') + self._counters.add_counter(f'statvar-for-depth-{depth}', 1) + return svg_grps + + def generate_groups_for_statvar(self, pvs: dict, svg_parent: str, + svg_prefix: str): + """Generates statvar groups for the hierarchy property:values in the statvar.""" + self._counters.add_counter('input-statvars', 1) + # Get the properties for the group + grp_props = dict() + for prop, value in pvs.items(): + prop = strip_namespace(prop) + value = strip_namespace(value) + ignore_val = strip_namespace(_DEFAULT_IGNORE_PROP.get(prop)) + if ignore_val is not None: + if not ignore_val or ignore_val == value: + continue + grp_props.setdefault(prop, value) + + # Get an ordered list of properties to create statvar groups. + # Also generate statvar for each set of properties. + leaf_svg = [] + linked_svgs = set() + strip_prefix = self._config.get('svg_dcid_remove_prefix', '') + for prop in self._config.get('svg_properties', ['populationType']): + val = grp_props.pop(prop, None) + if not val: + continue + val = re.sub(strip_prefix, '', to_snake_case(val)) + svg_dcid = svg_prefix + val + svg_name = self.get_schema_name(val) + self.add_statvar_group( + self.get_statvar_group_node(svg_dcid, svg_name, svg_parent)) + linked_svgs.add(add_namespace(svg_dcid)) + self._counters.add_counter(f'generated-statvar-groups-{prop}', 1) + svg_parent = svg_dcid + svg_prefix = svg_dcid + self._config.get('statvar_dcid_delimiter', + '__') + + # Generate statvar group for all permutations of properties. + if not grp_props: + leaf_svg.append(svg_parent) + linked_svgs.add(svg_parent) + props_perm = [sorted(list(grp_props.keys()))] + if self._config.get('statvar_group_permutations', False): + props_perm = list(itertools.permutations(props_perm[0])) + for props_list in props_perm: + parent_svgs = self.generate_prop_value_svg(pvs, props_list, + svg_parent, svg_prefix) + if parent_svgs: + leaf_svg.append(parent_svgs[-1]) + linked_svgs.update(parent_svgs) + + # Add the statvar to the leaf group. + sv = { + 'Node': add_namespace(get_node_dcid(pvs)), + 'typeOf': 'StatisticalVariable', + } + if leaf_svg: + sv['memberOf'] = ','.join( + [add_namespace(dcid) for dcid in leaf_svg]) + if linked_svgs and self._config.get('statvar_add_linked_member_of', + False): + sv['linkedMemberOf'] = ','.join( + [add_namespace(dcid) for dcid in linked_svgs]) + self.add_statvar_group(sv) + + def generate_statvar_groups(self, sv_nodes: dict): + """Generate statvar groups for given statvar nodes.""" + svg_prefix = self._config.get('svg_prefix', 'dc/g/') + svg_root = self._config.get('svg_root', 'dc/g/Root') + self._counters.add_counter('total', len(sv_nodes)) + for dcid, pvs in sv_nodes.items(): + self._counters.add_counter('processed', 1) + typ = strip_namespace(pvs.get('typeOf', '')) + if typ and typ != 'StatisticalVariable': + self._counters.add_counter('input-non-statvar-ignored', 1) + continue + self.generate_groups_for_statvar(pvs, svg_root, svg_prefix) + + # Make the top SVG a child of root + if 'Root' not in svg_root and self._config.get( + 'generate_statvar_group_root', True): + name = to_snake_case(svg_root[svg_root.rfind('/') + 1:], ' ', False) + self.add_statvar_group( + self.get_statvar_group_node(svg_root, name, 'dc/g/Root')) + self._counters.add_counter(f'generated-statvar-groups-root', 1) + + +def generate_statvar_groups(input_mcf: str, + schema_mcf: str, + output_mcf: str, + config: dict = None): + """Generate groups for statvars in input_mcf.""" + counters = Counters() + sv_grp_generator = UNStatVarGroupGenerator(config, counters) + sv_grp_generator.load_schema_mcf(schema_mcf) + + statvar_nodes = load_mcf_nodes(input_mcf) + logging.info(f'Generating statvar groups for {len(statvar_nodes)} nodes') + sv_grp_generator.generate_statvar_groups(statvar_nodes) + + sv_grps = sv_grp_generator.get_statvar_groups() + if output_mcf and sv_grps: + write_mcf_nodes(sv_grps, output_mcf) + counters.add_counter('output-nodes', len(sv_grps)) + + counters.print_counters() + + +def main(_): + logging.set_verbosity(_FLAGS.logging_level) + generate_statvar_groups(_FLAGS.input_statvar_mcf, _FLAGS.input_schema_mcf, + _FLAGS.output_statvar_group_mcf, + get_default_statvar_group_config()) + + +if __name__ == '__main__': + app.run(main) diff --git a/scripts/un/codes/generate_statvar_name.py b/scripts/un/codes/generate_statvar_name.py new file mode 100644 index 0000000000..60fbdec682 --- /dev/null +++ b/scripts/un/codes/generate_statvar_name.py @@ -0,0 +1,187 @@ +"""Script to generate statvar names for UN statvars.""" + +import os +import re +import sys + +from absl import app +from absl import flags +from absl import logging + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) +sys.path.append(os.path.dirname(_SCRIPT_DIR)) +sys.path.append(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +_DATA_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +sys.path.append(_DATA_DIR) +sys.path.append(os.path.join(_DATA_DIR, 'util')) +sys.path.append(os.path.join(_DATA_DIR, 'tools', 'statvar_importer')) + +import file_util +from mcf_file_util import add_namespace, strip_namespace, get_node_dcid +from mcf_file_util import load_mcf_nodes, write_mcf_nodes + +from config_map import ConfigMap +from counters import Counters + +flags.DEFINE_string('input_statvar_mcf', '', 'MCF files with statvar nodes.') +flags.DEFINE_string('output_statvar_mcf', '', + 'Output MCF files for statvar with names.') +flags.DEFINE_string('input_schema_mcf', '', + 'Schema file with names for properties.') +flags.DEFINE_integer('logging_level', logging.INFO, 'Logging level.') + +_FLAGS = flags.FLAGS +"""Wrapper to generate statvar names for UN StatVars.""" + +_DEFAULT_IGNORE_PROP = { + 'Node': '', + 'dcid': '', + 'typeOf': '', + 'memberOf': '', + 'footnote': '', + 'description': '', + 'name': '', + 'populationType': '', + 'measuredProperty': 'value', + 'statType': 'measuredValue', +} + +def to_quoted(text: str) -> str: + """Returns quoted string.""" + if not text: + return text + text = text.strip().strip('"').strip().replace('"', "'") + if text: + return '"' + text + '"' + return '' + +def to_sentence_case(text: str) -> str: + """Returns a string in sentence case.""" + # convert camelCase + sentence = re.sub(r'(?<=[a-z0-9])(?=[A-Z])', ' ', text) + + # convert '_' to spaces + sentence = re.sub(r'[_ ]+', ' ', sentence) + sentence = sentence.strip() + return sentence.capitalize() + + +class UNStatVarNameGenerator: + + def __init__( + self, + config_dict: dict = {}, + counters: Counters = None, + ): + self._config = ConfigMap() + self._config.update_config(config_dict) + self._counters = counters + if counters is None: + self._counters = Counters() + self._schema_nodes = {} + + def load_schema_mcf(self, mcf: str) -> dict: + """Loads schema nodes from MCF files.""" + load_mcf_nodes(mcf, nodes=self._schema_nodes) + self._counters.add_counter('input-schema-nodes', + len(self._schema_nodes)) + return self._schema_nodes + + def get_schema_node(self, dcid: str) -> dict: + """Returns a schema node for the dcid.""" + if not dcid: + return None + node = self._schema_nodes.get(strip_namespace(dcid)) + if not node: + node = self._schema_nodes.get(add_namespace(dcid)) + return node + + def get_schema_name(self, dcid: str) -> str: + """Returns the name for the dcid fomr the schema.""" + node = self.get_schema_node(dcid) + if not node: + return '' + name = node.get('alternateName') + if not name: + name = node.get('name', '') + return name.strip('"').strip() + + def generate_statvar_name(self, pvs: dict) -> dict: + """Adds a name to a statvar if it doesn't exist already.""" + name = pvs.get('name') + if name: + logging.debug(f'Using existing name for statvar:{name}') + self._counters.add_counter(f'input-existing-name', 1) + pvs['name'] = to_quoted(name) + return pvs + + # Use the name from the schema if it already exists. + dcid = get_node_dcid(pvs) + name = self.get_schema_name(dcid) + if name: + pvs['name'] = to_quoted(name) + self._counters.add_counter(f'input-schema-name', 1) + return pvs + + # Get the name from the populationType + name_prefix = self.get_schema_name(pvs.get('populationType')) + name_tokens = [] + # Collect names for constraint property:values + for prop, value in pvs.items(): + pv_tokens = [] + prop = strip_namespace(prop) + value = strip_namespace(value) + ignore_val = strip_namespace(_DEFAULT_IGNORE_PROP.get(prop)) + if ignore_val is not None: + if not ignore_val or ignore_val == value: + continue + prop_name = self.get_schema_name(prop) + if not prop_name: + prop_name = to_sentence_case(prop) + self._counters.add_counter('property-missing-name', 1) + if prop_name: + pv_tokens.append(prop_name) + val_name = self.get_schema_name(value) + if not val_name: + val_name = to_sentence_case(value) + self._counters.add_counter('value-missing-name', 1) + if val_name: + pv_tokens.append(val_name) + if pv_tokens: + name_tokens.append('='.join(pv_tokens)) + name_suffix = ', '.join(name_tokens) + name = name_prefix + if name_suffix: + self._counters.add_counter(f'generated-statvar-name-contraints', 1) + name = f'{name} [{name_suffix}]' + pvs['name'] = to_quoted(name) + self._counters.add_counter(f'generated-statvar-names', 1) + + +def generate_statvar_names(input_mcf: str, schema_mcf: str, output_mcf: str): + """Generate names for statvars in input_mcf.""" + counters = Counters() + config = {} + sv_name_generator = UNStatVarNameGenerator(config, counters) + sv_name_generator.load_schema_mcf(schema_mcf) + + statvar_nodes = load_mcf_nodes(input_mcf) + logging.info(f'Generating statvar names for {len(statvar_nodes)}') + for dcid, pvs in statvar_nodes.items(): + sv_name_generator.generate_statvar_name(pvs) + + if output_mcf: + write_mcf_nodes(statvar_nodes, output_mcf) + + counters.print_counters() + + +def main(_): + logging.set_verbosity(_FLAGS.logging_level) + generate_statvar_names(_FLAGS.input_statvar_mcf, _FLAGS.input_schema_mcf, + _FLAGS.output_statvar_mcf) + + +if __name__ == '__main__': + app.run(main) diff --git a/tools/statvar_importer/config_flags.py b/tools/statvar_importer/config_flags.py index a7449ca032..778fa96c52 100644 --- a/tools/statvar_importer/config_flags.py +++ b/tools/statvar_importer/config_flags.py @@ -169,7 +169,8 @@ 'Generate names for Statvars.') flags.DEFINE_bool('enable_cloud_logging', False, 'Enable cloud logging when running on cloud.') - +flags.DEFINE_string('statvar_dcid_prefix', '', + 'Prefix for statvar dcid.') def get_default_config() -> dict: """Returns the default config as dictionary of config parameters and values.""" @@ -433,6 +434,19 @@ def get_default_config() -> dict: _FLAGS.generate_statvar_name, # Generate names for StatVars 'llm_generate_statvar_name': _FLAGS.llm_generate_statvar_name, + + # Settings for statvar dcid generator + 'statvar_dcid_fixed_properties': [], + 'statvar_dcid_prefix': + _FLAGS.statvar_dcid_prefix, + 'statvar_dcid_remove_prefix': + '', + 'statvar_dcid_delimiter': + '', + 'statvar_dcid_value_delimiter': + '', + 'statvar_dcid_upper_case': + False, } diff --git a/tools/statvar_importer/schema/statvar_dcid_gen.py b/tools/statvar_importer/schema/statvar_dcid_gen.py new file mode 100644 index 0000000000..e32550be51 --- /dev/null +++ b/tools/statvar_importer/schema/statvar_dcid_gen.py @@ -0,0 +1,542 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities to generate statistical variable DCIDs from schema properties.""" + +import hashlib +import os +import re +import sys +from typing import Union + +from absl import app +from absl import flags +from absl import logging + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) +sys.path.append(os.path.dirname(_SCRIPT_DIR)) +sys.path.append(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +_DATA_DIR = os.path.dirname(os.path.dirname(os.path.dirname(_SCRIPT_DIR))) +sys.path.append(os.path.join(_DATA_DIR, 'util')) + +from counters import Counters +from dc_api_wrapper import dc_api_get_node_property_values +from mcf_file_util import ( + add_mcf_node, + add_namespace, + is_leaf_object, + strip_namespace, +) + +_RE_CAMEL_1 = re.compile(r'([a-z])([A-Z0-9])') +_RE_CAMEL_2 = re.compile(r'([A-Z])([A-Z][a-z])') +_RE_NON_ALNUM = re.compile(r'[^A-Za-z0-9_.-]+') +_RE_MULTI_UNDERSCORE = re.compile(r'_+') + + +def camel_to_snake(text: Union[str, None], delim: str = '_') -> str: + """Converts a string from camelCase to snake_case. + + Args: + text: The camelCase string to convert. + delim: Delimiter to use between words (default is '_'). + + Returns: + The converted snake_case string in lowercase. + + Example: + >>> camel_to_snake('camelCase') + 'camel_case' + """ + if not text: + return '' + text = str(text) + s1 = _RE_CAMEL_1.sub(r'\1' + delim + r'\2', text) + s2 = _RE_CAMEL_2.sub(r'\1' + delim + r'\2', s1) + return s2.lower() + + +def get_dcid_name( + dcid: Union[str, None], schema_nodes: Union[dict, None] +) -> Union[str, None]: + """Returns the human-readable name for a DCID from schema nodes if defined. + + Args: + dcid: The DCID string to look up. + schema_nodes: Dictionary of schema nodes containing properties. + + Returns: + The name of the DCID if found, or stripped DCID if no name property is + defined. Returns None if the DCID is not in the schema or invalid. + + Example: + >>> get_dcid_name('dcid:Count', {'Count': {'name': '"Total Count"'}}) + 'Total Count' + """ + if not dcid or schema_nodes is None: + return None + dcid_str = str(dcid) + node = schema_nodes.get(strip_namespace(dcid_str)) + if not node: + node = schema_nodes.get(add_namespace(dcid_str)) + if not node or not isinstance(node, dict): + return None + name = node.get('name') + if not name: + name = strip_namespace(dcid_str) + return str(name).strip().strip('"').strip() + + +def get_dcid_token(word: Union[str, None], + upper_case: bool = False, + remove_prefix: str = '') -> str: + """Returns the word normalized into a token suitable for a DCID. + + Args: + word: The raw string to normalize. + upper_case: If True, converts camelCase to uppercase snake_case. + remove_prefix: Optional prefix or regex pattern to remove from token. + + Returns: + A normalized DCID token string with alphanumeric chars and underscores. + + Example: + >>> get_dcid_token('helloWorld', upper_case=True) + 'HELLO_WORLD' + """ + if not word: + return '' + # Convert any non-alphanumeric characters to '_' + token = _RE_NON_ALNUM.sub('_', str(word).strip()) + token = _RE_MULTI_UNDERSCORE.sub('_', token).strip('_') + + if upper_case: + # Convert camelCase to snake_case + token = camel_to_snake(token).upper() + if remove_prefix and token: + try: + token = re.sub(remove_prefix, '', token) + except re.error as e: + logging.warning( + f'Invalid regex prefix "{remove_prefix}" in get_dcid_token: {e}' + ) + if token.startswith(remove_prefix): + token = token[len(remove_prefix):] + if token: + token = token.strip('_.-') + if token and any(c.isalnum() for c in token): + return token[0].upper() + token[1:] + return '' + + +def parse_fixed_properties( + dcid_props: Union[list[str], tuple[str, ...], None] = None +) -> dict[str, set[str]]: + """Parses fixed property definitions and their ignored default values. + + Args: + dcid_props: List of property strings, optionally containing '<>' + to specify ignored values (e.g. 'statType<>measuredValue'). + + Returns: + A dictionary mapping property names to sets of ignored values. + + Example: + >>> parse_fixed_properties( + ... ['statType<>measuredValue', 'populationType'] + ... ) + {'statType': {'measuredValue'}, 'populationType': {''}} + """ + if not dcid_props: + dcid_props = [ + 'statType<>measuredValue', + 'measurementQualifier', + 'measuredProperty', + 'populationType', + ] + fixed_props = dict() + for prop_spec in dcid_props: + if not prop_spec or not isinstance(prop_spec, str): + continue + prop = prop_spec.strip() + val = '' + if '<>' in prop: + prop, val = prop.split('<>', 1) + prop = prop.strip() + val = val.strip() + if prop: + fixed_props.setdefault(prop, set()).add(val) + return fixed_props + + +def resolve_dcid_names(pvs: dict, + schema_nodes: dict, + ignore_props: set[str], + use_value_names: bool = False, + counters: Counters = None) -> dict[str, str]: + """Filters ignored properties and fetches missing node names via DC API. + + Args: + pvs: Source property-value dictionary for the statistical variable. + schema_nodes: Dictionary of loaded schema nodes to query and update. + ignore_props: Set of property names to exclude from DCID generation. + use_value_names: Whether to resolve human-readable names for DCIDs. + counters: Optional `Counters` object to record lookup statistics. + + Returns: + A filtered dictionary of property-value pairs to include in the DCID. + """ + dcid_pvs = dict() + lookup_dcids = set() + for prop, value in pvs.items(): + if not prop or prop in ignore_props: + continue + dcid_pvs[prop] = value + if use_value_names: + if not get_dcid_name(prop, schema_nodes): + lookup_dcids.add(prop) + if value and is_leaf_object(value) and not get_dcid_name( + value, schema_nodes + ): + lookup_dcids.add(value) + + if lookup_dcids: + if counters: + counters.add_counter('dc_api_lookup_name', len(lookup_dcids)) + try: + node_names = dc_api_get_node_property_values(list(lookup_dcids)) + if node_names: + for node_pvs in node_names.values(): + if node_pvs: + add_mcf_node(node_pvs, schema_nodes) + except Exception as e: + logging.error( + f'Failed fetching node names for DCIDs {lookup_dcids}: {e}' + ) + return dcid_pvs + + +def strip_overlapping_prop_prefix(value_token: Union[str, None], + prop: Union[str, None], + upper_case: bool = False) -> str: + """Removes property name prefix from a value token if overlapping. + + If `value_token` starts with the property name (in camelCase, snake_case, + or uppercase) followed by a delimiter or capital letter, the common prefix + is removed so that only the distinct value/code remains. + + Args: + value_token: The normalized token representing the property value. + prop: The property name string (e.g. 'measurementQualifier'). + upper_case: Whether tokens are being formatted in uppercase. + + Returns: + The `value_token` with any overlapping property prefix removed. + + Example: + >>> strip_overlapping_prop_prefix( + ... 'MeasurementQualifier_Annual', 'measurementQualifier' + ... ) + 'Annual' + >>> strip_overlapping_prop_prefix( + ... 'UNIT_PERCENT', 'unit', upper_case=True + ... ) + 'PERCENT' + """ + if not value_token or not prop: + return value_token or '' + + val_str = str(value_token) + prop_str = str(prop).strip() + if not prop_str or len(val_str) <= len(prop_str): + return val_str + + # Generate candidate prefixes representing the property string + candidates = set() + c_token = get_dcid_token(prop_str, upper_case=upper_case) + if c_token: + candidates.add(c_token) + c_upper = get_dcid_token(prop_str, upper_case=True) + if c_upper: + candidates.add(c_upper) + c_title = get_dcid_token(prop_str, upper_case=False) + if c_title: + candidates.add(c_title) + + snake = camel_to_snake(prop_str) + if snake: + candidates.add(snake) + candidates.add(snake.upper()) + candidates.add(snake.replace('_', '')) + candidates.add(snake.replace('_', '').upper()) + + # Sort candidates by length descending to strip the longest matching prefix + sorted_candidates = sorted( + [c for c in candidates if c and len(val_str) > len(c)], + key=len, + reverse=True, + ) + + for cand in sorted_candidates: + if val_str.lower().startswith(cand.lower()): + idx = len(cand) + next_char = val_str[idx] + if (next_char in ('_', '-', '.', ':') or + next_char.isupper() or upper_case): + remainder = val_str[idx:].lstrip('_.-:') + if remainder: + if upper_case: + return remainder.upper() + return remainder[0].upper() + remainder[1:] + return val_str + + +def order_dcid_properties( + dcid_pvs: dict[str, str], fixed_props: dict[str, set[str]] +) -> list[str]: + """Orders DCID properties, dropping fixed properties with ignored values. + + Fixed properties from the configuration are ordered first in their defined + sequence, unless value matches an ignored default (e.g. measuredValue). + All remaining properties are appended in alphabetical order. + + Args: + dcid_pvs: Filtered property-value dictionary (modified in-place if + ignored values are popped). + fixed_props: Mapping of fixed property names to sets of ignored values. + + Returns: + Ordered list of property names to be tokenized into the DCID. + """ + ordered_props = [] + for prop, ignored_vals in fixed_props.items(): + prop_val = dcid_pvs.get(prop) + if prop_val: + if ignored_vals and prop_val in ignored_vals: + dcid_pvs.pop(prop, None) + else: + ordered_props.append(prop) + for prop in sorted(dcid_pvs.keys()): + if prop not in ordered_props: + ordered_props.append(prop) + return ordered_props + + +def _tokenize_dcid( + ordered_props: list[str], + dcid_pvs: dict[str, str], + fixed_props: dict[str, set[str]], + use_value_names: bool, + config: dict, + schema_nodes: dict, +) -> tuple[str, list[tuple[str, str]], list[tuple[str, str]]]: + """Helper to tokenize properties into fixed and constraint token pairs.""" + dcid_prefix = config.get('statvar_dcid_prefix', '') + prop_delim = config.get('statvar_dcid_delimiter', '_') + fixed_prop_delim = config.get('statvar_dcid_fixed_delimiter', '_') + val_delim = config.get('statvar_dcid_value_delimiter', '') + upper_case = config.get('statvar_dcid_upper_case', False) + remove_prefix = config.get('statvar_dcid_remove_prefix', '') + + fixed_pairs = [] + prop_pairs = [] + dcid_fixed_tokens = [] + dcid_prop_tokens = [] + + for prop in ordered_props: + prop_value = dcid_pvs.get(prop) + if not prop_value: + continue + value_name = prop_value + if use_value_names: + value_name = get_dcid_name(prop_value, schema_nodes) or prop_value + value_name = get_dcid_token(value_name, upper_case, remove_prefix) + value_name = strip_overlapping_prop_prefix( + value_name, prop, upper_case=upper_case + ) + if val_delim and prop not in fixed_props: + prop_name = get_dcid_token(prop, upper_case) + value_name = prop_name + val_delim + value_name + if upper_case: + value_name = value_name.upper() + if prop in fixed_props: + fixed_pairs.append((prop, value_name)) + dcid_fixed_tokens.append(value_name) + else: + prop_pairs.append((prop, value_name)) + dcid_prop_tokens.append(value_name) + + prop_token = prop_delim.join(dcid_prop_tokens) + if prop_token: + dcid_fixed_tokens.append(prop_token) + dcid = fixed_prop_delim.join(dcid_fixed_tokens) + if dcid_prefix: + dcid = dcid_prefix + dcid + return dcid, fixed_pairs, prop_pairs + + +def apply_cumulative_property_dropping( + fixed_pairs: list[tuple[str, str]], + prop_pairs: list[tuple[str, str]], + max_len: int, + config: dict, +) -> str: + """Drops secondary constraints incrementally to stay within max_len. + + Retains core fixed properties and as many secondary constraint properties + as can fit within `max_len`. Omitted properties are deterministically + hashed and appended as a short hash suffix to preserve uniqueness. + + Args: + fixed_pairs: List of (property, token) tuples for core properties. + prop_pairs: List of (property, token) tuples for secondary constraints. + max_len: Maximum allowed character length for the DCID string. + config: Configuration dictionary with delimiters and prefix settings. + + Returns: + The truncated DCID string ending with a deterministic hash suffix. + """ + dcid_prefix = config.get('statvar_dcid_prefix', '') + prop_delim = config.get('statvar_dcid_delimiter', '_') + fixed_prop_delim = config.get('statvar_dcid_fixed_delimiter', '_') + hash_len = config.get('statvar_dcid_hash_length', 8) + + all_tokens = [t for _, t in fixed_pairs + prop_pairs] + full_str = fixed_prop_delim.join(all_tokens) + full_hash = hashlib.md5(full_str.encode('utf-8')).hexdigest()[:hash_len] + full_hash = full_hash.upper() + + retained_fixed = [] + retained_props = [] + omitted = False + + for prop, token in fixed_pairs: + test_fixed = retained_fixed + [token] + test_str = fixed_prop_delim.join(test_fixed) + if dcid_prefix: + test_str = dcid_prefix + test_str + if len(test_str) + len(fixed_prop_delim) + hash_len <= max_len: + retained_fixed.append(token) + else: + omitted = True + break + + if not omitted: + for prop, token in prop_pairs: + test_props = retained_props + [token] + prop_token = prop_delim.join(test_props) + test_fixed = retained_fixed + [prop_token] + test_str = fixed_prop_delim.join(test_fixed) + if dcid_prefix: + test_str = dcid_prefix + test_str + if len(test_str) + len(fixed_prop_delim) + hash_len <= max_len: + retained_props.append(token) + else: + omitted = True + break + + final_fixed = list(retained_fixed) + if retained_props: + final_fixed.append(prop_delim.join(retained_props)) + if omitted or not final_fixed: + final_fixed.append(full_hash) + + dcid = fixed_prop_delim.join(final_fixed) + if dcid_prefix: + dcid = dcid_prefix + dcid + if len(dcid) > max_len: + budget = max(0, max_len - hash_len - 1) + clean_prefix = dcid[:budget].rstrip('_.-:/') if budget > 0 else '' + if clean_prefix: + return f"{clean_prefix}_{full_hash}" + return full_hash[:max_len] + return dcid + + +def generate_dcid_for_statvar(pvs: Union[dict, None], + config: Union[dict, None] = None, + schema_nodes: Union[dict, None] = None, + counters: Union[Counters, None] = None) -> str: + """Generates a statistical variable DCID from property-value mappings. + + Args: + pvs: Dictionary of property-value mappings representing the StatVar. + config: Configuration dictionary defining DCID generation parameters. + schema_nodes: Optional dictionary of loaded schema nodes. + counters: Optional `Counters` object to track statistics. + + Returns: + A generated DCID string for the StatVar. + + Example: + >>> generate_dcid_for_statvar( + ... {'statType': 'measuredValue', 'measuredProperty': 'count', + ... 'populationType': 'Person'}, + ... {} + ... ) + 'Count_Person' + """ + if not pvs or not isinstance(pvs, dict): + return '' + if config is None or not isinstance(config, dict): + config = {} + if schema_nodes is None or not isinstance(schema_nodes, dict): + schema_nodes = {} + + # Parse fixed properties and their default/ignored values from config + fixed_props = parse_fixed_properties( + config.get('statvar_dcid_fixed_properties') + ) + + use_value_names = config.get('statvar_dcid_value_name', False) + ignore_props = set( + config.get('statvar_dcid_ignore_properties', [ + 'description', 'name', 'nameWithLanguage', 'descriptionUrl', + 'alternateName', 'footnote', 'unCode', 'Node', 'typeOf' + ]) + ) + + # Filter ignored properties and resolve missing DCID names via DC API + dcid_pvs = resolve_dcid_names( + pvs, schema_nodes, ignore_props, use_value_names, counters + ) + + # Order fixed properties first followed by sorted constraint properties + ordered_props = order_dcid_properties(dcid_pvs, fixed_props) + + # Tokenize each property value into standardized DCID components + dcid, fixed_pairs, prop_pairs = _tokenize_dcid( + ordered_props, dcid_pvs, fixed_props, use_value_names, config, + schema_nodes + ) + + max_len = config.get('statvar_dcid_max_length', 255) + if max_len > 0 and len(dcid) > max_len: + # Fallback Strategy #3: Retry with raw codes instead of long value names + if use_value_names: + if counters: + counters.add_counter('statvar_dcid_fallback_raw_codes', 1) + dcid, fixed_pairs, prop_pairs = _tokenize_dcid( + ordered_props, dcid_pvs, fixed_props, False, config, + schema_nodes + ) + # Fallback Strategy #4: Cumulative prioritized property dropping + hash + if len(dcid) > max_len: + if counters: + counters.add_counter('statvar_dcid_truncated_with_hash', 1) + dcid = apply_cumulative_property_dropping( + fixed_pairs, prop_pairs, max_len, config + ) + + return dcid diff --git a/tools/statvar_importer/schema/statvar_dcid_gen_test.py b/tools/statvar_importer/schema/statvar_dcid_gen_test.py new file mode 100644 index 0000000000..8440583bdb --- /dev/null +++ b/tools/statvar_importer/schema/statvar_dcid_gen_test.py @@ -0,0 +1,256 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from statvar_dcid_gen import apply_cumulative_property_dropping +from statvar_dcid_gen import camel_to_snake +from statvar_dcid_gen import generate_dcid_for_statvar +from statvar_dcid_gen import get_dcid_name +from statvar_dcid_gen import get_dcid_token +from statvar_dcid_gen import order_dcid_properties +from statvar_dcid_gen import parse_fixed_properties +from statvar_dcid_gen import resolve_dcid_names +from statvar_dcid_gen import strip_overlapping_prop_prefix + + +class TestStatvarDcidGen(unittest.TestCase): + + def test_camel_to_snake(self): + self.assertEqual(camel_to_snake('camelCase'), 'camel_case') + self.assertEqual(camel_to_snake('CamelCase'), 'camel_case') + self.assertEqual(camel_to_snake('CaseACRONYM'), 'case_acronym') + self.assertEqual(camel_to_snake('CaseAbc123'), 'case_abc_123') + self.assertEqual(camel_to_snake('simple'), 'simple') + + def test_get_dcid_token(self): + self.assertEqual(get_dcid_token('Hello World!'), 'Hello_World') + self.assertEqual(get_dcid_token('helloWorld', upper_case=True), + 'HELLO_WORLD') + self.assertEqual(get_dcid_token('prefixWorld', remove_prefix='prefix'), + 'World') + + def test_get_dcid_name(self): + schema_nodes = { + 'Person': { + 'name': '"Human"' + }, + 'dcid:Count': { + 'name': 'TotalCount' + }, + } + self.assertEqual(get_dcid_name('Person', schema_nodes), 'Human') + self.assertEqual(get_dcid_name('dcid:Person', schema_nodes), 'Human') + self.assertEqual(get_dcid_name('Count', schema_nodes), 'TotalCount') + self.assertEqual(get_dcid_name('Unknown', schema_nodes), None) + + def test_generate_dcid(self): + pvs = { + 'statType': 'measuredValue', + 'measuredProperty': 'count', + 'populationType': 'Person', + } + dcid = generate_dcid_for_statvar(pvs, {}) + self.assertEqual(dcid, 'Count_Person') + + pvs2 = { + 'statType': 'index', + 'measuredProperty': 'count', + 'populationType': 'Person', + } + dcid2 = generate_dcid_for_statvar(pvs2, {}) + self.assertEqual(dcid2, 'Index_Count_Person') + + def test_generate_dcid_with_property(self): + config = { + 'statvar_dcid_fixed_properties': [ + 'statType<>measuredValue', 'measuredProperty<>value', + 'populationType' + ], + 'statvar_dcid_delimiter': '__', + 'statvar_dcid_fixed_delimiter': '.', + 'statvar_dcid_value_delimiter': '--', + 'statvar_dcid_remove_prefix': 'TEST_', + 'statvar_dcid_upper_case': True, + 'statvar_dcid_prefix': 'test/', + } + pvs = { + 'statType': 'measuredValue', + 'measuredProperty': 'count', + 'populationType': 'Person', + } + dcid = generate_dcid_for_statvar(pvs, config) + self.assertEqual(dcid, 'test/COUNT.PERSON') + + pvs2 = { + 'statType': 'medianValue', + 'measuredProperty': 'age', + 'populationType': 'Person', + 'gender': 'Male', + 'place': 'TEST_Urban', + } + dcid2 = generate_dcid_for_statvar(pvs2, config) + self.assertEqual( + dcid2, 'test/MEDIAN_VALUE.AGE.PERSON.GENDER--MALE__PLACE--URBAN') + pvs3 = { + 'statType': 'measuredValue', + 'measuredProperty': 'value', + 'populationType': 'AdultPerson', + 'gender': 'Male', + 'place': 'TEST_Urban', + } + dcid3 = generate_dcid_for_statvar(pvs3, config) + self.assertEqual( + dcid3, 'test/ADULT_PERSON.GENDER--MALE__PLACE--URBAN' + ) + + def test_parse_fixed_properties(self): + props = ['statType<>measuredValue', 'populationType'] + res = parse_fixed_properties(props) + self.assertEqual( + res, {'statType': {'measuredValue'}, 'populationType': {''}} + ) + default_res = parse_fixed_properties(None) + self.assertIn('statType', default_res) + + def test_order_dcid_properties(self): + dcid_pvs = {'statType': 'measuredValue', 'gender': 'Female'} + fixed = {'statType': {'measuredValue'}} + ordered = order_dcid_properties(dcid_pvs, fixed) + self.assertEqual(ordered, ['gender']) + self.assertNotIn('statType', dcid_pvs) + + def test_resolve_dcid_names(self): + pvs = {'statType': 'measuredValue', 'description': 'Ignore me'} + schema_nodes = {} + resolved = resolve_dcid_names(pvs, schema_nodes, {'description'}) + self.assertEqual(resolved, {'statType': 'measuredValue'}) + + def test_edge_cases(self): + self.assertEqual(camel_to_snake(None), '') + self.assertIsNone(get_dcid_name(None, {})) + self.assertEqual(get_dcid_token(None), '') + self.assertEqual(generate_dcid_for_statvar(None), '') + + def test_strip_overlapping_prop_prefix(self): + self.assertEqual( + strip_overlapping_prop_prefix( + 'MeasurementQualifier_Annual', 'measurementQualifier' + ), + 'Annual', + ) + self.assertEqual( + strip_overlapping_prop_prefix( + 'UNIT_PERCENT', 'unit', upper_case=True + ), + 'PERCENT', + ) + self.assertEqual( + strip_overlapping_prop_prefix('Person', 'populationType'), + 'Person', + ) + + def test_generate_dcid_with_overlapping_prefix(self): + pvs = { + 'statType': 'measuredValue', + 'measuredProperty': 'count', + 'populationType': 'Person', + 'measurementQualifier': 'MeasurementQualifier_Annual', + } + dcid = generate_dcid_for_statvar(pvs, {}) + self.assertEqual(dcid, 'Annual_Count_Person') + + def test_dcid_max_length_fallbacks(self): + # Test #3 fallback: use_value_names falls back to raw code when > max + schema_nodes = { + 'Count': { + 'name': '"ExtremelyLongDescriptiveCountNameOfManyWords"' + } + } + pvs = { + 'statType': 'measuredValue', + 'measuredProperty': 'Count', + 'populationType': 'Person', + } + config = { + 'statvar_dcid_value_name': True, + 'statvar_dcid_max_length': 30, + } + dcid = generate_dcid_for_statvar(pvs, config, schema_nodes) + self.assertEqual(dcid, 'Count_Person') + + # Test #4 fallback: cumulative dropping + deterministic hash + long_pvs = { + 'statType': 'measuredValue', + 'measuredProperty': 'count', + 'populationType': 'Person', + 'age': 'Age_0_To_18_Years_Old', + 'gender': 'Gender_Female_Or_Male', + 'place': 'Place_California_Or_New_York', + } + short_config = {'statvar_dcid_max_length': 45} + dcid_hashed = generate_dcid_for_statvar(long_pvs, short_config) + self.assertTrue(len(dcid_hashed) <= 45) + self.assertTrue(dcid_hashed.startswith('Count_Person')) + self.assertIn('_', dcid_hashed) + + def test_complex_edge_cases(self): + # 1. camel_to_snake complex cases + self.assertEqual( + camel_to_snake('HTTP2ServerResponse', delim='-'), + 'http2server-response', + ) + self.assertEqual(camel_to_snake('ALREADY_SNAKE'), 'already_snake') + + # 2. get_dcid_name complex cases + schema_nodes = { + 'Person': {'name': ' " Complex Person Name " '}, + 'Count': {'typeOf': 'Property'}, + 'BadNode': 'NotADict', + } + self.assertEqual( + get_dcid_name('un:Person', schema_nodes), 'Complex Person Name' + ) + self.assertEqual(get_dcid_name('Count', schema_nodes), 'Count') + self.assertIsNone(get_dcid_name('BadNode', schema_nodes)) + + # 3. get_dcid_token complex cases (invalid regex warning recovery) + self.assertEqual( + get_dcid_token('Hello World!', remove_prefix='['), 'Hello_World' + ) + self.assertEqual( + get_dcid_token('TEST_prefix_value', remove_prefix='TEST_'), + 'Prefix_value', + ) + self.assertEqual(get_dcid_token('...___...'), '') + + # 4. parse_fixed_properties complex cases + props = ['statType<>measuredValue<>extra', ' populationType <> ', 12] + parsed = parse_fixed_properties(props) + self.assertEqual( + parsed, + {'statType': {'measuredValue<>extra'}, 'populationType': {''}}, + ) + + # 5. apply_cumulative_property_dropping boundary cases + small_drop = apply_cumulative_property_dropping( + [('p1', 'VERY_LONG_CORE')], + [('p2', 'CONSTRAINT')], + max_len=5, + config={'statvar_dcid_hash_length': 8}, + ) + self.assertEqual(len(small_drop), 5) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/statvar_importer/stat_var_processor.py b/tools/statvar_importer/stat_var_processor.py index 487cce556f..2767a18d02 100644 --- a/tools/statvar_importer/stat_var_processor.py +++ b/tools/statvar_importer/stat_var_processor.py @@ -89,6 +89,7 @@ from schema_generator import generate_schema_nodes, generate_statvar_name from schema_checker import sanity_check_nodes from schema_reconciler import SchemaReconciler +from statvar_dcid_gen import generate_dcid_for_statvar # imports from ../../util from config_map import ConfigMap, read_py_dict_from_file @@ -375,14 +376,19 @@ def generate_statvar_dcid(self, pvs: dict) -> str: 'statvar_dcid_ignore_properties', [ 'description', 'name', 'nameWithLanguage', 'descriptionUrl', - 'alternateName' + 'alternateName', 'footnote', 'typeOf', 'Node' ], ) if not self._config.get( 'schemaless', False) or not self._get_schemaless_statvar_props(pvs): try: - dcid = get_statvar_dcid(pvs, ignore_props=dcid_ignore_props) + if self._config.get('statvar_dcid_fixed_properties'): + # Use the custom statvar dcid generator + dcid = generate_dcid_for_statvar(pvs, self._config, + self._counters) + else: + dcid = get_statvar_dcid(pvs, ignore_props=dcid_ignore_props) dcid = re.sub(r'[^A-Za-z_0-9/_\.-]+', '_', dcid) except TypeError as e: logging.log_every_n( @@ -1216,6 +1222,7 @@ def format_svobs(self, svobs: dict) -> dict: numeric_value, precision_digits=self._config.get('output_precision_digits', 5), + max_int=self._config.get('max_integer', sys.maxsize), ) elif isinstance(value, str) and value: value = value.strip() diff --git a/tools/statvar_importer/utils.py b/tools/statvar_importer/utils.py index 9c060de0b0..17c98832a6 100644 --- a/tools/statvar_importer/utils.py +++ b/tools/statvar_importer/utils.py @@ -67,7 +67,8 @@ def capitalize_first_char(string: str) -> str: def str_from_number(number: Union[int, float], - precision_digits: Optional[int] = None) -> str: + precision_digits: Optional[int] = None, + max_int: int = sys.maxsize) -> str: """Converts a number (int or float) to its string representation. Integers and floats that are whole numbers (e.g., 10.0) are returned as @@ -77,6 +78,7 @@ def str_from_number(number: Union[int, float], Args: number: The number to convert. precision_digits: Optional number of decimal places to round a float to. + max_int: Numbers larger than this are converted to float Returns: The string representation of the number. @@ -94,7 +96,10 @@ def str_from_number(number: Union[int, float], '123.45' """ # Check if number is an integer or float without any decimals. - if int(number) == number: + if abs(number) > max_int: + # Convert very large ints to float with potential loss of precision + number = float(number) + elif int(number) == number: number_int = int(number) return f'{number_int}' # Return float rounded to precision digits.