diff --git a/statvar_imports/mongolia_imports/common_download_script.py b/statvar_imports/mongolia_imports/common_download_script.py index 17254046b4..0197831b5b 100644 --- a/statvar_imports/mongolia_imports/common_download_script.py +++ b/statvar_imports/mongolia_imports/common_download_script.py @@ -64,7 +64,7 @@ "filename": "resident_population_by_agegroup_15_and_over_and_maritalstatus.csv", "header_mapping": { - "keys": ["SCR_ENG", "SCR_ENG1", "SCR_ENG2", "CODE"], + "keys": ["SCR_ENG2", "SCR_ENG1", "SCR_ENG", "CODE"], "cols": ["Marital Status", "Age Group", "Gender"] } }, @@ -72,8 +72,8 @@ "id": "DT_NSO_0300_004V1", "filename": "total_population_by_region_and_urban_rural.csv", "header_mapping": { - "keys": ["SCR_ENG", "SCR_ENG1", "CODE1", "CODE"], - "cols": ["Total", "Aimag", "Код"] + "keys": ["SCR_ENG", "SCR_ENG1", "CODE"], + "cols": ["Aimag", "Total", "Код"] } }, { @@ -81,7 +81,7 @@ "filename": "number_of_households_by_region_and_urban_rural.csv", "header_mapping": { "keys": ["SCR_ENG", "SCR_ENG1", "CODE"], - "cols": ["NUMBER OF HOUSEHOLDS", "Aimag", "Код"] + "cols": ["Aimag", "NUMBER OF HOUSEHOLDS", "Код"] } }, ] @@ -183,6 +183,81 @@ }] + + + + +_API_CACHE = {} + +def find_table_path(session, table_id): + import re + tid = table_id.upper() + if tid.endswith(".PX"): + tid = tid[:-3] + + # Base ID for fuzzy version match (e.g., DT_NSO_0300_077V1 -> DT_NSO_0300_077) + base_match = re.match(r"(DT_NSO_\d+_\d+)V\d+", tid) + base_id = base_match.group(1) if base_match else tid + + def get_cached_json(url): + if url not in _API_CACHE: + try: + r = session.get(url, timeout=60) + if r.status_code == 200: + _API_CACHE[url] = r.json() + else: + _API_CACHE[url] = None + except Exception as e: + logging.error(f"Failed to fetch {url}: {e}") + _API_CACHE[url] = None + return _API_CACHE[url] + + sectors_url = "https://data.1212.mn/api/v1/en/NSO" + sectors = get_cached_json(sectors_url) + if not sectors: + return None + + candidates = [] + for sector in sectors: + sector_id = sector.get("id") + if not sector_id: + continue + subsectors_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}" + subsectors = get_cached_json(subsectors_url) + if not subsectors: + continue + + for subsector in subsectors: + subsector_id = subsector.get("id") + if not subsector_id: + continue + tables_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}" + tables = get_cached_json(tables_url) + if not tables: + continue + + for table in tables: + table_item_id = table.get("id", "").upper() + if table_item_id.endswith(".PX"): + table_item_id = table_item_id[:-3] + + # Exact match + if table_item_id == tid: + return sector_id, subsector_id, table.get("id") + + # Fuzzy version match candidate + if table_item_id.startswith(base_id): + candidates.append((sector_id, subsector_id, table.get("id"), table_item_id)) + + if candidates: + # Sort candidates so that we get the highest version first or exact version if possible + candidates.sort(key=lambda x: x[3], reverse=True) + logging.info(f"Fuzzy version match found for {table_id}: selected {candidates[0][2]}") + return candidates[0][0], candidates[0][1], candidates[0][2] + + return None + + def fetch_and_save_data(table_id, csv_filepath, header_mapping): """ Fetches data from the API for a given table ID, pivots it, @@ -196,95 +271,185 @@ def fetch_and_save_data(table_id, csv_filepath, header_mapping): """ logging.info(f"Processing {table_id} -> {csv_filepath}...") - url = "https://opendata.1212.mn/api/Data" - data = {"tbl_id": table_id} - headers = {"Content-Type": "application/json"} - retry_logic = Retry(total=5, backoff_factor=1, allowed_methods=["POST"]) + retry_logic = Retry(total=10, backoff_factor=1, allowed_methods=["GET", "POST"]) adapter = HTTPAdapter(max_retries=retry_logic) session = requests.Session() session.mount("https://", adapter) try: - response = session.post(url, headers=headers, data=json.dumps(data)) + # 1. Resolve table path + path_info = find_table_path(session, table_id) + if not path_info: + error_msg = f"FATAL ERROR for {table_id}: Table not found in API catalog. Aborting." + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + sector_id, subsector_id, table_filename = path_info + logging.info(f"Resolved path for {table_id}: NSO/{sector_id}/{subsector_id}/{table_filename}") + + # 2. Fetch metadata (English and Mongolian) + en_meta_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}/{table_filename}" + mn_meta_url = f"https://data.1212.mn/api/v1/mn/NSO/{sector_id}/{subsector_id}/{table_filename}" + + en_meta_res = session.get(en_meta_url, timeout=60) + mn_meta_res = session.get(mn_meta_url, timeout=60) + + if en_meta_res.status_code != 200 or mn_meta_res.status_code != 200: + error_msg = f"FATAL ERROR for {table_id}: Failed to fetch metadata (EN: {en_meta_res.status_code}, MN: {mn_meta_res.status_code})" + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + en_meta = en_meta_res.json() + mn_meta = mn_meta_res.json() + + # 3. Fetch all data using POST + data_url = f"https://data.1212.mn/api/v1/en/NSO/{sector_id}/{subsector_id}/{table_filename}" + payload = { + "query": [], + "response": { + "format": "json" + } + } + headers = {"Content-Type": "application/json"} + response = session.post(data_url, headers=headers, json=payload, timeout=60) # Check status code first if response.status_code == 200: response_data = response.json() - if not response_data: - # Checks if the dictionary is empty[from the source we are getting data in dictionary fomate] + if not response_data or "data" not in response_data: error_msg = f"FATAL ERROR for {table_id}: No data found in the source. Aborting script." logging.fatal(error_msg) raise RuntimeError(error_msg) logging.info("Success! Response data received.") - if "DataList" in response_data and isinstance( - response_data["DataList"], list): - data_list = response_data["DataList"] - - # Check for empty DataList and log the finding before aborting - if not data_list: - logging.info( - f"Found 'DataList' structure in source, but it is empty: {table_id}." - ) - error_msg = f"FATAL ERROR for {table_id}: 'DataList' is present but contains zero records. No data found from the Source" - logging.fatal(error_msg) - raise RuntimeError(error_msg) - if len(data_list) < 40: # Updated threshold to 40 records - logging.info( - f"Found 'DataList' structure in source, but it is empty or too small: {table_id}. DataList length: {len(data_list)}" - ) - error_msg = f"FATAL ERROR for {table_id}: 'DataList' contains less than 40 records. Data is not sufficient." - logging.fatal(error_msg) - raise RuntimeError(error_msg) - - pivoted_data = {} - all_periods = set() - - for item in data_list: - period = item.get("Period", "") - row_keys = [ - item.get(key, "") for key in header_mapping['keys'] - ] - dtval_co = item.get("DTVAL_CO", "") - row_key = tuple(row_keys) - - if period: - all_periods.add(period) - if row_key not in pivoted_data: - pivoted_data[row_key] = {} - pivoted_data[row_key][period] = dtval_co - - sorted_periods = sorted(list(all_periods)) - - try: - with open(csv_filepath, 'w', newline='', - encoding='utf-8') as csvfile: - csv_writer = csv.writer(csvfile) - - csv_headers = header_mapping['cols'] + sorted_periods - csv_writer.writerow(csv_headers) - - for row_key, period_values in pivoted_data.items(): - - # It creates the row using only as many keys as there are column names. - row = list(row_key)[:len(header_mapping['cols'])] - - for period in sorted_periods: - row.append(period_values.get(period, "")) - csv_writer.writerow(row) - - logging.info( - f"Successfully created CSV file: {csv_filepath}\n") - except IOError as e: - error_msg = f"Failed to write CSV file {csv_filepath}: {e}" - logging.fatal(error_msg) - raise RuntimeError(error_msg) - else: - logging.warning( - f"Error for {table_id}: 'DataList' not found in the API response.\n" + # Map variables + variables = en_meta.get("variables", []) + period_var_idx = -1 + for idx, var in enumerate(variables): + code_lower = var.get("code", "").lower() if var else "" + text_lower = var.get("text", "").lower() if var else "" + if any(p in code_lower or p in text_lower for p in ["year", "period", "time", "month", "он", "сар"]): + period_var_idx = idx + break + + class_vars = [] + for idx, (en_v, mn_v) in enumerate(zip(en_meta.get("variables", []), mn_meta.get("variables", []))): + if idx != period_var_idx: + class_vars.append((idx, en_v, mn_v)) + + N = len(class_vars) + + # Pre-build lookups for period and class variables to avoid O(L) index lookups in the loop + period_lookup = {} + if period_var_idx != -1: + en_var = en_meta.get("variables", [])[period_var_idx] if period_var_idx < len(en_meta.get("variables", [])) else None + if en_var: + en_values = en_var.get("values", []) + en_texts = en_var.get("valueTexts", []) + for idx, val in enumerate(en_values): + period_lookup[val] = en_texts[idx] if idx < len(en_texts) else val + + class_var_lookups = [] + for var_idx, en_v, mn_v in class_vars: + en_values = en_v.get("values", []) if en_v else [] + en_texts = en_v.get("valueTexts", []) if en_v else [] + mn_texts = mn_v.get("valueTexts", []) if mn_v else [] + lookup = {} + for idx, val in enumerate(en_values): + en_t = en_texts[idx] if idx < len(en_texts) else val + mn_t = mn_texts[idx] if idx < len(mn_texts) else val + lookup[val] = (en_t, mn_t) + class_var_lookups.append((var_idx, lookup)) + + import re + data_list = [] + for item in response_data.get("data", []): + keys = item.get("key", []) + vals = item.get("values", []) + if not vals: + continue + + row_data = {} + row_data["DTVAL_CO"] = vals[0] + + if period_var_idx != -1 and period_var_idx < len(keys): + period_key = keys[period_var_idx] + p_val = period_lookup.get(period_key, period_key) + if re.match(r"^\d{4}-\d{2}$", p_val): + p_val = p_val.replace("-", "") + row_data["Period"] = p_val + else: + row_data["Period"] = "" + + for i, (var_idx, lookup) in enumerate(class_var_lookups): + suffix = str(N - 1 - i) if i < N - 1 else "" + key_val = keys[var_idx] if var_idx < len(keys) else "" + en_text, mn_text = lookup.get(key_val, (key_val, key_val)) + row_data[f"CODE{suffix}"] = key_val + row_data[f"SCR_ENG{suffix}"] = en_text + row_data[f"SCR_MN{suffix}"] = mn_text + data_list.append(row_data) + + # Check for empty DataList and log the finding before aborting + if not data_list: + logging.info( + f"Found empty DataList for: {table_id}." + ) + error_msg = f"FATAL ERROR for {table_id}: DataList contains zero records. No data found from the Source" + logging.fatal(error_msg) + raise RuntimeError(error_msg) + if len(data_list) < 40: # Updated threshold to 40 records + logging.info( + f"Found too small DataList: {table_id}. DataList length: {len(data_list)}" ) + error_msg = f"FATAL ERROR for {table_id}: DataList contains less than 40 records. Data is not sufficient." + logging.fatal(error_msg) + raise RuntimeError(error_msg) + + pivoted_data = {} + all_periods = set() + + for item in data_list: + period = item.get("Period", "") + row_keys = [ + str(item.get(key, "")).strip() for key in header_mapping['keys'] + ] + dtval_co = item.get("DTVAL_CO", "") + row_key = tuple(row_keys) + + if period: + all_periods.add(period) + if row_key not in pivoted_data: + pivoted_data[row_key] = {} + pivoted_data[row_key][period] = dtval_co + + sorted_periods = sorted(list(all_periods)) + + try: + with open(csv_filepath, 'w', newline='', + encoding='utf-8') as csvfile: + csv_writer = csv.writer(csvfile) + + csv_headers = header_mapping['cols'] + sorted_periods + csv_writer.writerow(csv_headers) + + for row_key, period_values in pivoted_data.items(): + + # It creates the row using only as many keys as there are column names. + row = list(row_key)[:len(header_mapping['cols'])] + + for period in sorted_periods: + row.append(period_values.get(period, "")) + csv_writer.writerow(row) + + logging.info( + f"Successfully created CSV file: {csv_filepath}\n") + except IOError as e: + error_msg = f"Failed to write CSV file {csv_filepath}: {e}" + logging.fatal(error_msg) + raise RuntimeError(error_msg) else: logging.warning( f"Error for {table_id}: Request failed with status code {response.status_code}\n" diff --git a/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv index 28652958e5..8d5589d6be 100644 --- a/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_demographics/resident_population_by_agegroup_15_and_over_and_maritalstatus_pvmap.csv @@ -10,13 +10,22 @@ Single,maritalStatus,Unmarried,,,,,,,, 2000,observationDate,2000,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count 2010,observationDate,2010,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count 2015,observationDate,2015,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2020,observationDate,2020,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2021,observationDate,2021,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2022,observationDate,2022,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2023,observationDate,2023,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2025,observationDate,2025,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2026,observationDate,2026,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2027,observationDate,2027,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2028,observationDate,2028,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2029,observationDate,2029,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count +2030,observationDate,2030,value,{Number},observationAbout,country/MNG,populationType,Person,measuredProperty,count ,,,,,,,,,, -Age Group:Population aged 15 and over-Total,age,[15 - Years],,,,,,,, -Age Group:15-19,#ignore,skip,,,,,,,, -Age Group:70+,#ignore,skip,,,,,,,, +Age Group:Total,age,[15 - Years],,,,,,,, +Age Group:20-29,age,[20 29 Years],,,,,,,, +Age Group:30-39,age,[30 39 Years],,,,,,,, +Age Group:40-49,age,[40 49 Years],,,,,,,, +Age Group:50-59,age,[50 59 Years],,,,,,,, +Age Group:60-69,age,[60 69 Years],,,,,,,, ,,,,,,,,,, -20-29,age,[20 29 Years],,,,,,,, -30-39,age,[30 39 Years],,,,,,,, -40-49,age,[40 49 Years],,,,,,,, -50-59,age,[50 59 Years],,,,,,,, -60-69,age,[60 69 Years],,,,,,,, diff --git a/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv index fce3110dcf..8d7d79e8ae 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/graduates_of_universities_and_colleges_by_professional_field_pvmap.csv @@ -13,7 +13,7 @@ Biological and related sciences,bachelorsDegreeMajor,BiologicalAndBiomedicalScie Environment,bachelorsDegreeMajor,EnvironmentMajor,,,,,, Natural sciences,bachelorsDegreeMajor,NaturalSciencesMajor,,,,,, Mathematics and statistics,bachelorsDegreeMajor,MathAndStatisticsMajor,,,,,, -Information and Communication Technologies,bachelorsDegreeMajor,InformationAndCommunicationsTechnologiesMajor ,,,,,, +Information and Communication Technologies,bachelorsDegreeMajor,InformationAndCommunicationsTechnologiesMajor,,,,,, "Engineering, manufacturing and construction",bachelorsDegreeMajor,EngineeringMajor__EngineeringManufacturing__ConstructionEngineering,,,,,, Architecture and construction,bachelorsDegreeMajor,ArchitectureAndRelatedServicesMajor__ConstructionEngineering,,,,,, Engineering between industries,bachelorsDegreeMajor,EngineeringMajor__Industries,,,,,, @@ -22,6 +22,7 @@ Production and processing,bachelorsDegreeMajor,ProductionEngineering__Processing "Agriculture, forestry, fisheries and veterinary",bachelorsDegreeMajor,AgriculturalEngineering__ForestryEngineering__Fisheries__Veterinary,,,,,, Veterinary medicine,bachelorsDegreeMajor,VeterinaryMedicineMajor,,,,,, "Agriculture, forestry, fisheries",bachelorsDegreeMajor,AgriculturalEngineering__ForestryEngineering__Fisheries,,,,,, +Health and welfare,bachelorsDegreeMajor,HealthRelatedMajor,,,,,, Health and social protection,bachelorsDegreeMajor,HealthRelatedMajor,,,,,, Oral and Maxillofacial Studies,bachelorsDegreeMajor,OralAndMaxillofacialStudies,,,,,, Nursing and midwifery,bachelorsDegreeMajor,NursingAndMidwifery,,,,,, diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv index 34d2396aaf..5097ed1606 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv @@ -28,4 +28,10 @@ Number of teacher:Total,populationType,Teacher,measuredProperty,count,employment 2021,observationDate,2021,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count 2022,observationDate,2022,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count 2023,observationDate,2023,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count -2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count \ No newline at end of file +2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2025,observationDate,2025,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2026,observationDate,2026,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2027,observationDate,2027,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2028,observationDate,2028,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2029,observationDate,2029,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count +2030,observationDate,2030,value,{Number},observationAbout,country/MNG,populationType,Teacher,measuredProperty,count \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv index fc29252b7f..f13fa3a559 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_kindergartens_by_region_pvmap.csv @@ -24,4 +24,10 @@ Aimag,observationAbout,{Data},schoolGradeLevel,Kindergarten,populationType,Schoo 2021,observationDate,2021,value,{Number},,,, 2022,observationDate,2022,value,{Number},,,, 2023,observationDate,2023,value,{Number},,,, -2024,observationDate,2024,value,{Number},,,, \ No newline at end of file +2024,observationDate,2024,value,{Number},,,, +2025,observationDate,2025,value,{Number},,,, +2026,observationDate,2026,value,{Number},,,, +2027,observationDate,2027,value,{Number},,,, +2028,observationDate,2028,value,{Number},,,, +2029,observationDate,2029,value,{Number},,,, +2030,observationDate,2030,value,{Number},,,, \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv index 3e1eb07f2b..81b33f9e67 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/number_of_students_in_universities_and_colleges_by_region_pvmap.csv @@ -19,4 +19,10 @@ Female,gender,Female,,,, 2021,observationDate,2021,value,{Number},, 2022,observationDate,2022,value,{Number},, 2023,observationDate,2023,value,{Number},, -2024,observationDate,2024,value,{Number},, \ No newline at end of file +2024,observationDate,2024,value,{Number},, +2025,observationDate,2025,value,{Number},, +2026,observationDate,2026,value,{Number},, +2027,observationDate,2027,value,{Number},, +2028,observationDate,2028,value,{Number},, +2029,observationDate,2029,value,{Number},, +2030,observationDate,2030,value,{Number},, \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_education/run.sh b/statvar_imports/mongolia_imports/mongolia_education/run.sh index 73abefc34c..43438347f9 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/run.sh +++ b/statvar_imports/mongolia_imports/mongolia_education/run.sh @@ -9,7 +9,7 @@ python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --inp python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_students_in_universities_and_colleges_by_region.csv --pv_map=$SCRIPT_PATH/number_of_students_in_universities_and_colleges_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_students_in_universities_and_colleges_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/mongolia_place_resolver.csv || { echo "Error: Processing number_of_students_in_universities_and_colleges_by_region.csv failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_kindergartens_by_region.csv --pv_map=$SCRIPT_PATH/number_of_kindergartens_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_kindergartens_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing number_of_kindergartens_by_region_pvmap.csv failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_kindergartens_by_region.csv --pv_map=$SCRIPT_PATH/number_of_kindergartens_by_region_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_kindergartens_by_region_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --places_resolved_csv=$SCRIPT_PATH/mongolia_place_resolver.csv || { echo "Error: Processing number_of_kindergartens_by_region_pvmap.csv failed!"; exit 1; } python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex.csv --pv_map=$SCRIPT_PATH/number_of_full_time_teachers_in_universities_and_colleges_by_sex_pvmap.csv --config_file=$SCRIPT_PATH/mongolia_metadata.csv --output_path=$SCRIPT_PATH/output_files/number_of_full_time_teachers_in_universities_and_colleges_by_sex_output --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf || { echo "Error: Processing number_of_full_time_teachers_in_universities_and_colleges_by_sex.csv failed!"; exit 1; } diff --git a/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv index 2b14b206fb..19574afe3f 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/students_in_teritary_educational_institutions_by_sex_and_educational_degree_pvmap.csv @@ -2,6 +2,7 @@ key,p1,v1,p2,v2,, ,,,,,, Professional fields,schoolGradeLevel,TertiaryEducation,populationType,Student,observationAbout,country/MNG Of which: Female,gender,Female,schoolGradeLevel,TertiaryEducation,, +Bachelor,enrollmentLevel,BachelorsDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Bachelor¹,enrollmentLevel,BachelorsDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Master,enrollmentLevel,MastersDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel Ph.D,enrollmentLevel,DoctorateDegree,schoolGradeLevel,TertiaryEducation,#Header,enrollmentLevel @@ -30,3 +31,9 @@ Upper diploma,enrollmentLevel,UpperDiploma,schoolGradeLevel,TertiaryEducation,#H 2022,observationDate,2022,value,{Number},, 2023,observationDate,2023,value,{Number},, 2024,observationDate,2024,value,{Number},, +2025,observationDate,2025,value,{Number},, +2026,observationDate,2026,value,{Number},, +2027,observationDate,2027,value,{Number},, +2028,observationDate,2028,value,{Number},, +2029,observationDate,2029,value,{Number},, +2030,observationDate,2030,value,{Number},, diff --git a/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv b/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv index 4341f3efca..33ac607fa6 100644 --- a/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv +++ b/statvar_imports/mongolia_imports/mongolia_education/students_of_universities_and_colleges_by_professional_field_pvmap.csv @@ -29,4 +29,10 @@ General basic curriculum,bachelorsDegreeMajor,GeneralBasicCurriculumMajor,,,,,,, 2021,observationDate,2021,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool 2022,observationDate,2022,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool 2023,observationDate,2023,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool -2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool \ No newline at end of file +2024,observationDate,2024,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2025,observationDate,2025,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2026,observationDate,2026,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2027,observationDate,2027,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2028,observationDate,2028,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2029,observationDate,2029,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool +2030,observationDate,2030,value,{Number},observationAbout,country/MNG,populationType,Student,collegeOrGraduateSchoolEnrollment,EnrolledInCollegeOrGraduateSchool \ No newline at end of file diff --git a/statvar_imports/mongolia_imports/mongolia_employment/places_resolved.csv b/statvar_imports/mongolia_imports/mongolia_employment/places_resolved.csv index 54cf6d3d07..d014c8917c 100644 --- a/statvar_imports/mongolia_imports/mongolia_employment/places_resolved.csv +++ b/statvar_imports/mongolia_imports/mongolia_employment/places_resolved.csv @@ -1,28 +1,135 @@ place_name,dcid -, -Western region, -Bayan-Ulgii,wikidataId/Q191792 -Govi-Altai,wikidataId/Q192945 -Zavkhan,wikidataId/Q167764 -Uvs,wikidataId/Q192942 -Khovd,wikidataId/Q194098 -Khangai region, + Arkhangai,wikidataId/Q207809 + Arkhangai,wikidataId/Q207809 + Arkhangai,wikidataId/Q207809 Arkhangai,wikidataId/Q207809 + Baganuur Bagakhangai,NA + Baganuur Bagakhangai,NA + Bayan-Ulgii,wikidataId/Q191792 + Bayan-Ulgii,wikidataId/Q191792 + Bayan-Ulgii,wikidataId/Q191792 +Bayan-Ulgii,wikidataId/Q191792 + Bayangol,NA + Bayangol,NA + Bayankhongor,wikidataId/Q276200 + Bayankhongor,wikidataId/Q276200 + Bayankhongor,wikidataId/Q276200 Bayankhongor,wikidataId/Q276200 + Bayanzurkh,NA + Bayanzurkh,NA + Bulgan,wikidataId/Q209774 + Bulgan,wikidataId/Q209774 + Bulgan,wikidataId/Q209774 Bulgan,wikidataId/Q209774 -Orkhon,wikidataId/Q234710 -Uvurkhangai,wikidataId/Q234713 -Khuvsgul,wikidataId/Q244788 -Central region, -Govisumber,wikidataId/Q236333 + Central region,NA + Central region,NA +Central region,NA +Centrel,NA + Chingeltei,NA + Chingeltei,NA + Darkhan-Uul,wikidataId/Q18827 + Darkhan-Uul,wikidataId/Q18827 + Darkhan-Uul,wikidataId/Q18827 Darkhan-Uul,wikidataId/Q18827 + Dornod,wikidataId/Q207795 + Dornod,wikidataId/Q207795 + Dornod,wikidataId/Q207795 +Dornod,wikidataId/Q207795 + Dornogovi,wikidataId/Q213272 + Dornogovi,wikidataId/Q213272 + Dornogovi,wikidataId/Q213272 Dornogovi,wikidataId/Q213272 + Dundgovi,wikidataId/Q211835 + Dundgovi,wikidataId/Q211835 + Dundgovi,wikidataId/Q211835 Dundgovi,wikidataId/Q211835 -Umnugovi,wikidataId/Q235579 +Eastern,NA + Eastern region,NA + Eastern region,NA +Eastern region,NA + Govi-Altai,wikidataId/Q192945 + Govi-Altai,wikidataId/Q192945 + Govi-Altai,wikidataId/Q192945 +Govi-Altai,wikidataId/Q192945 + Govisumber,wikidataId/Q236333 + Govisumber,wikidataId/Q236333 + Govisumber,wikidataId/Q236333 +Govisumber,wikidataId/Q236333 + Khan-Uul,NA + Khan-Uul,NA +Khangai,NA + Khangai region,NA + Khangai region,NA +Khangai region,NA + Khentii,wikidataId/Q239040 + Khentii,wikidataId/Q239040 + Khentii,wikidataId/Q239040 +Khentii,wikidataId/Q239040 + Khovd,wikidataId/Q194098 + Khovd,wikidataId/Q194098 + Khovd,wikidataId/Q194098 +Khovd,wikidataId/Q194098 + Khuvsgul,wikidataId/Q244788 + Khuvsgul,wikidataId/Q244788 + Khuvsgul,wikidataId/Q244788 +Khuvsgul,wikidataId/Q244788 + Nalaikh,NA + Nalaikh,NA +National result,country/MNG + Orkhon,wikidataId/Q234710 + Orkhon,wikidataId/Q234710 + Orkhon,wikidataId/Q234710 +Orkhon,wikidataId/Q234710 + Selenge,wikidataId/Q234680 + Selenge,wikidataId/Q234680 + Selenge,wikidataId/Q234680 Selenge,wikidataId/Q234680 -Tuv,wikidataId/Q276195 -Eastern region, -Dornod,wikidataId/Q207795 + Songinokhairkhan,NA + Songinokhairkhan,NA + Sukhbaatar,wikidataId/Q244804 + Sukhbaatar,wikidataId/Q244804 + Sukhbaatar,wikidataId/Q244804 + Sukhbaatar,wikidataId/Q244804 + Sukhbaatar,wikidataId/Q244804 Sukhbaatar,wikidataId/Q244804 -Khentii,wikidataId/Q239040 +Total,country/MNG + Tuv,wikidataId/Q276195 + Tuv,wikidataId/Q276195 + Tuv,wikidataId/Q276195 +Tuv,wikidataId/Q276195 + Ulaanbaatar,wikidataId/Q23430 + Ulaanbaatar,wikidataId/Q23430 + Ulaanbaatar,wikidataId/Q23430 + Ulaanbaatar,wikidataId/Q23430 Ulaanbaatar,wikidataId/Q23430 + Umnugovi,wikidataId/Q235579 + Umnugovi,wikidataId/Q235579 + Umnugovi,wikidataId/Q235579 +Umnugovi,wikidataId/Q235579 + Uvs,wikidataId/Q192942 + Uvs,wikidataId/Q192942 + Uvs,wikidataId/Q192942 +Uvs,wikidataId/Q192942 + Uvurkhangai,wikidataId/Q234713 + Uvurkhangai,wikidataId/Q234713 + Uvurkhangai,wikidataId/Q234713 +Uvurkhangai,wikidataId/Q234713 +Western,NA + Western region,NA + Western region,NA +Western region,NA + Zavkhan,wikidataId/Q167764 + Zavkhan,wikidataId/Q167764 + Zavkhan,wikidataId/Q167764 +Zavkhan,wikidataId/Q167764 +Багануур Багахангай,NA +Баянгол,NA +Баянзүрх,NA +Налайх,NA +Сонгинохайрхан,NA + Сүхбаатар,wikidataId/Q244804 +Сүхбаатар,wikidataId/Q244804 + Улаанбаатар,wikidataId/Q23430 +Улаанбаатар,wikidataId/Q23430 +Хан-Уул,NA +Чингэлтэй,NA