S1201 schema fix and data refresh

import-automation/executor/requirements.txt

@@ -1,6 +1,7 @@
 # Requirements for Python scripts in this repo that have automation enabled!
 
 absl-py
+aiolimiter
 arcgis2geojson
 beautifulsoup4
 chardet

scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py

@@ -25,11 +25,17 @@
 from collections import OrderedDict
 import pandas as pd
 
-# Allows the following module imports to work when running as a script
+# Allows the sibling/parent module imports to work when running as a script or module
 _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(_SCRIPT_PATH,
-                             '../common'))  # for statvar_dcid_generator
-from generate_col_map import generate_stat_var_map
+_curr = _SCRIPT_PATH
+while _curr and _curr != os.path.dirname(_curr):
+    if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists(
+            os.path.join(_curr, 'WORKSPACE')):
+        sys.path.append(_curr)
+        break
+    _curr = os.path.dirname(_curr)
+
+from scripts.us_census.acs5yr.subject_tables.common.generate_col_map import generate_stat_var_map
 
 
 def process_zip_file(zip_file_path,

scripts/us_census/acs5yr/subject_tables/s1201/process.py

@@ -12,23 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Generic proces module to generate the csv/tmcf and csv"""
-# TODO: Add unit tests
+# Allows the sibling/parent module imports to work when running as a script or module
 import os
 import sys
+
+_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
+_curr = _SCRIPT_PATH
+while _curr and _curr != os.path.dirname(_curr):
+    if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists(
+            os.path.join(_curr, 'WORKSPACE')):
+        sys.path.append(_curr)
+        break
+    _curr = os.path.dirname(_curr)
+
+# TODO: Add unit tests
 import json
 from zipfile import ZipFile
 import pandas as pd
 
 from absl import app, flags
 # TODO: logs from the column map step is empty when invoked from here, needs to be checked
 
-from .generate_col_map import generate_stat_var_map, process_zip_file
-
-# Allows the following module imports to work when running as a script
-_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(_SCRIPT_PATH,
-                             '../common'))  # for col_map_generator, data_loader
-from data_loader import process_subject_tables
+from scripts.us_census.acs5yr.subject_tables.s1201.generate_col_map import process_zip_file
+from scripts.us_census.acs5yr.subject_tables.common.data_loader import process_subject_tables
 
 FLAGS = flags.FLAGS
 flags.DEFINE_string(
@@ -56,9 +62,8 @@ def set_column_map(input_path, spec_path, output_dir):
     generated_col_map = process_zip_file(input_path,
                                          spec_path,
                                          write_output=False)
-    f = open(os.path.join(output_dir, 'column_map.json'), 'w')
-    json.dump(generated_col_map, f, indent=4)
-    f.close()
+    with open(os.path.join(output_dir, 'column_map.json'), 'w') as f:
+        json.dump(generated_col_map, f, indent=4)
 
 
 def main(argv):
@@ -71,6 +76,8 @@ def main(argv):
     has_percent = FLAGS.has_percent
     debug = FLAGS.debug
 
+    os.makedirs(output_dir, exist_ok=True)
+
     # TODO: remove the constraint of inputs being only zip file
     # context: the current implementation of the column map generator accepts
     # only zip files as input and we will need to add new methods to handle inputs

scripts/us_census/api_utils/census_api_config_fetcher.py

@@ -29,9 +29,9 @@
 CONFIG_PATH_ = os.path.join(module_dir_, 'config_files')
 path.insert(1, os.path.join(module_dir_, '../../../'))
 
-from .download_utils import download_url_list_iterations, async_save_resp_json
+from scripts.us_census.api_utils.download_utils import download_url_list_iterations, async_save_resp_json
 from tools.download_utils.requests_wrappers import request_url_json
-from .status_file_utils import get_pending_or_fail_url_list, sync_status_list
+from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, sync_status_list
 
 FLAGS = flags.FLAGS
 

scripts/us_census/api_utils/census_api_data_downloader.py

@@ -35,9 +35,9 @@
 module_dir_ = os.path.dirname(os.path.realpath(__file__))
 path.insert(1, os.path.join(module_dir_, '../../../'))
 
-from .download_utils import download_url_list_iterations
+from scripts.us_census.api_utils.download_utils import download_url_list_iterations
 from tools.download_utils.requests_wrappers import request_url_json
-from .status_file_utils import sync_status_list
+from scripts.us_census.api_utils.status_file_utils import sync_status_list
 
 FLAGS = flags.FLAGS
 
@@ -300,6 +300,7 @@ def consolidate_files(dataset: str,
                 # substitute annotations
                 if table_id in column_name and column_name[-1] != 'A':
                     if replace_annotations:
+                        df2[column_name] = df2[column_name].astype(object)
                         df2.loc[df2[column_name + 'A'].notna(),
                                 column_name] = df2[column_name + 'A']
                     if drop_annotations:

scripts/us_census/api_utils/download_utils.py

@@ -24,7 +24,7 @@
 from typing import Any, Callable, Union
 from aiolimiter import AsyncLimiter
 
-from .status_file_utils import get_pending_or_fail_url_list, url_to_download
+from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, url_to_download
 
 
 async def async_save_resp_json(response: Any, filename: str):

scripts/us_census/api_utils/download_utils_test.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import unittest
-from .download_utils import *
+
+from scripts.us_census.api_utils.download_utils import async_save_resp_json, download_url_list
 
 
 class TestCommonUtil(unittest.TestCase):
@@ -50,4 +52,4 @@ def test_download_url_list(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

scripts/us_census/api_utils/status_file_utils_test.py

@@ -12,10 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import unittest
 
-from .status_file_utils import *
+from scripts.us_census.api_utils.status_file_utils import (
+    get_failed_http_url_list,
+    get_failed_url_list,
+    get_pending_or_fail_url_list,
+    get_pending_url_list,
+    url_to_download,
+)
 
 
 class TestCommonUtil(unittest.TestCase):
@@ -108,4 +115,4 @@ def test_get_pending_or_fail_url_list(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

scripts/us_census/api_utils/url_list_compiler.py

@@ -26,7 +26,7 @@
 from typing import Any, Union
 
 from census_api_helpers import *
-from .status_file_utils import sync_status_list
+from scripts.us_census.api_utils.status_file_utils import sync_status_list
 
 FLAGS = flags.FLAGS
 

util/dc_api_wrapper_test.py

@@ -244,8 +244,19 @@ def test_dc_api_resolve_placeid(self):
         self.assertEqual(response['ChIJT3IGqvxznW4Rqgw7pv9zYz8'],
                          'wikidataId/Q9727')
 
-    def test_dc_api_resolve_latlng(self):
+    @mock.patch('dc_api_wrapper.request_url')
+    def test_dc_api_resolve_latlng(self, mock_request_url):
         """Test API wrapper for latlng resolution."""
+        mock_request_url.return_value = {
+            "entities": [{
+                "node":
+                    "37.42#-122.08",
+                "candidates": [{
+                    "dcid": "geoId/0649670",
+                    "dominantType": "City"
+                }]
+            }]
+        }
         latlngs = [{'latitude': 37.42, 'longitude': -122.08}]
         response = dc_api.dc_api_resolve_latlng(latlngs)
 

util/latlng_recon_service_test.py

@@ -34,7 +34,36 @@ def assert_list_contains(self, superset_list, subset_list):
             f"The following items were expected but not found: {sorted(list(missing_items))}"
         )
 
-    def test_basic(self):
+    @mock.patch('util.latlng_recon_service.dc_api_resolve_latlng')
+    def test_basic(self, mock_dc_api_resolve_latlng):
+        mock_dc_api_resolve_latlng.return_value = {
+            'placeCoordinates': [{
+                'latitude':
+                    37.391,
+                'longitude':
+                    -122.081,
+                'placeDcids': [
+                    'zip/94041', 'ipcc_50/37.25_-122.25_USA',
+                    'geoId/sch0626280', 'geoId/0649670', 'geoId/0618',
+                    'geoId/0608592830', 'geoId/060855096001',
+                    'geoId/06085509600', 'geoId/06085', 'geoId/06',
+                    'country/USA'
+                ]
+            }, {
+                'latitude':
+                    12.998,
+                'longitude':
+                    80.272,
+                'placeDcids': [
+                    'wikidataId/Q15116', 'wikidataId/Q1445',
+                    'ipcc_50/12.75_80.25_IND', 'country/IND'
+                ]
+            }, {
+                'latitude': 37.700,
+                'longitude': -123.015,
+                'placeDcids': []
+            }]
+        }
         idmap_in = {
             'cascal_mtv': (37.391, -122.081),
             'besant_beach_chennai': (12.998, 80.272),
@@ -53,7 +82,27 @@ def test_basic(self):
         ])
         self.assertEqual(idmap_out['farallon_islands'], [])
 
-    def test_filter(self):
+    @mock.patch('util.latlng_recon_service.dc_api_resolve_latlng')
+    def test_filter(self, mock_dc_api_resolve_latlng):
+        mock_dc_api_resolve_latlng.return_value = {
+            'placeCoordinates': [{
+                'latitude':
+                    37.391,
+                'longitude':
+                    -122.081,
+                'placeDcids': [
+                    'zip/94041', 'ipcc_50/37.25_-122.25_USA',
+                    'geoId/sch0626280', 'geoId/0649670', 'geoId/0618',
+                    'geoId/0608592830', 'geoId/060855096001',
+                    'geoId/06085509600', 'geoId/06085', 'geoId/06',
+                    'country/USA'
+                ]
+            }, {
+                'latitude': 37.700,
+                'longitude': -123.015,
+                'placeDcids': []
+            }]
+        }
         idmap_in = {
             'cascal_mtv': (37.391, -122.081),
             'farallon_islands': (37.700, -123.015)